+
    œ°³jód  ã                   óP  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIHtHt ^RIHtHt ^RIH t H!t! ^RI"H#t# ^RI$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t. ^RI/H0t0 ]! R4       ! R R]Pb                  4      4       t2 ! R R]Pb                  4      t3 ! R R]Pb                  4      t4R t5]! R4      R6R l4       t6R R lt7R7R  R! llt8]! ]64       ! R" R#]Pb                  4      4       t9R$ t:]! R%R&4      R8R' R( ll4       t;]! R)R&4      R8R* R+ ll4       t<]! ];]<.4       ! R, R-]Pb                  4      4       t= ! R. R/]4      t>]& ! R0 R1]!4      4       t?]& ! R2 R3]?4      4       t@]& ! R4 R5]?]4      4       tA. R9OtBR# ):é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚ&use_kernel_func_from_hub_with_fallbackÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
Lfm2ConfigÚRMSNormc                   óZ   a a€ ] tR t^,t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚLfm2RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/lfm2/modeling_lfm2.pyÚ__annotate__ÚLfm2RMSNorm.__annotate__.   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z*
Lfm2RMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer%   Ú	__class__s   &&&€r*   r0   ÚLfm2RMSNorm.__init__.   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör-   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r$   Úhidden_statesr&   )r2   ÚTensor)r(   r)   s   "€r*   r+   r,   6   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r-   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r$   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor2   Úfloat32ÚpowÚmeanÚrsqrtr5   r4   )r6   r;   Úinput_dtypeÚvariances   &&  r*   ÚforwardÚLfm2RMSNorm.forward6   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r-   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler4   Úshaper5   )r6   s   &r*   Ú
extra_reprÚLfm2RMSNorm.extra_repr=   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr-   )r5   r4   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r0   rH   rM   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r8   r)   s   @@r*   r"   r"   ,   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr-   r"   c                   óÎ   a a€ ] tR t^At o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚLfm2RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r$   Úconfigr   )r(   r)   s   "€r*   r+   Ú Lfm2RotaryEmbedding.__annotate__C   s   ø€ ÷ Oñ O™zñ Or-   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r/   r0   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr]   Úrope_parametersr`   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r6   r]   rY   Úrope_init_fnrj   r8   s   &&&  €r*   r0   ÚLfm2RotaryEmbedding.__init__B   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr-   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r]   r&   )r   rK   r2   r<   r'   )r(   r)   s   "€r*   r+   r^   U   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r-   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©r@   )rf   Úgetattrr7   Únum_attention_headsr2   Úaranger'   rA   )r]   rY   ÚkwargsÚbaseÚdimÚattention_factorrj   s   &&,    r*   rg   Ú3Lfm2RotaryEmbedding.compute_default_rope_parametersS   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r-   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©ry   rs   )NºNNNNr?   )r‚   Nr‚   )rj   r'   ÚexpandrL   rA   rY   Ú
isinstanceÚtypeÚstrr   Ú	transposer2   ÚcatÚcosrh   Úsinr@   )
r6   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr   ÚfreqsÚembr‰   rŠ   s
   &&&       r*   rH   ÚLfm2RotaryEmbedding.forwardg   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)rh   r]   rj   rd   rl   re   r`   ©N)rO   rP   rQ   rR   r   r0   Ústaticmethodrg   r2   Úno_gradr   rH   rS   rT   rU   rV   s   @@r*   rX   rX   A   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r-   rX   c                   óD   a a€ ] tR t^wt oV3R lV 3R lltR tRtVtV ;t# )ÚLfm2MLPc                ó    <€ V ^8„  d   QhRS[ /# r\   r   )r(   r)   s   "€r*   r+   ÚLfm2MLP.__annotate__x   s   ø€ ÷ Oñ O™zñ Or-   c                óJ  <€ \         SV `  4        VP                  pVP                  '       d   \	        ^V,          ^,          4      pVP
                  eX   \	        VP
                  V,          4      pVP                  W!P                  ,           ^,
          VP                  ,          ,          p\        P                  ! VP                  VRR7      V n
        \        P                  ! VP                  VRR7      V n        \        P                  ! W!P                  RR7      V n        R# )r$   NF©Úbias)r/   r0   Úintermediate_sizeÚblock_auto_adjust_ff_dimÚintÚblock_ffn_dim_multiplierÚblock_multiple_ofr   ÚLinearr7   Úw1Úw3Úw2)r6   r]   rœ   r8   s   && €r*   r0   ÚLfm2MLP.__init__x   sÝ   ø€ Ü‰ÑÔØ"×4Ñ4ÐØ×*×*Ð*Ü # AÐ(9Õ$9¸AÕ$=Ó >Ðà×.Ñ.Ò:Ü$'¨×(GÑ(GÐJ[Õ([Ó$\Ð!Ø$*×$<Ñ$<Ø&×)AÑ)AÕAÀAÕEÈ&×JbÑJbÕbõ%Ð!ô —)’)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)’)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)’)Ð-×/AÑ/AÈÔNˆŽr-   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # r’   )r¤   ÚFÚsilur¢   r£   )r6   r‹   s   &&r*   rH   ÚLfm2MLP.forward‡   s/   € Ø�w‰w”q—v’v˜dŸg™g a›jÓ)¨D¯G©G°A«JÕ6Ó7Ð7r-   )r¢   r¤   r£   ©	rO   rP   rQ   rR   r0   rH   rS   rT   rU   rV   s   @@r*   r–   r–   w   s   ù‡ € ÷Oó O÷8ò 8r-   r–   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr�   r?   )rL   r2   rˆ   )r‹   Úx1Úx2s   &  r*   Úrotate_halfr®   ‹   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r-   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer®   )ÚqÚkr‰   rŠ   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r*   Úapply_rotary_pos_embr·   ’   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr-   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   r;   Ún_repr&   )r2   r<   rž   )r(   s   "r*   r+   r+   ¬   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur-   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r‚   r‚   Nr‚   r‚   )rL   rƒ   Úreshape)r;   r¹   ÚbatchÚnum_key_value_headsÚslenrr   s   &&    r*   Ú	repeat_kvr¿   ¬   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr-   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrw   )r   ÚModuler2   r<   r'   r   r   )r(   s   "r*   r+   r+   ¸   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r-   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r$   )ry   r@   )ÚpÚtrainingr?   )r¿   Únum_key_value_groupsr2   Úmatmulr‡   r   Ú
functionalÚsoftmaxrB   rA   r@   rÇ   rË   Ú
contiguous)rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   rÇ   rw   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r*   Úeager_attention_forwardrÕ   ¸   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r-   c                   óX   a a€ ] tR t^Ñt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚLfm2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r$   r]   Ú	layer_idx©r   rž   )r(   r)   s   "€r*   r+   ÚLfm2Attention.__annotate__Õ   s   ø€ ÷ Kñ K™zð K±cñ Kr-   c                óÈ  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        \%        V P                  VP&                  R7      V n        \%        V P                  VP&                  R7      V n        R# )rr   TFrš   ©r%   Ng      à¿)r/   r0   r]   rÚ   rt   r7   ru   rr   r½   rÌ   rÆ   Ú	is_causalr   r¡   Úq_projÚk_projÚv_projÚout_projr"   Únorm_epsÚq_layernormÚk_layernorm©r6   r]   rÚ   r8   s   &&&€r*   r0   ÚLfm2Attention.__init__Õ   sH  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜŸ	š	 &×"<Ñ"<¸t¿}¹}Õ"LÈf×N`ÑN`ÐglÔmˆŒÜ& t§}¡}¸&¿/¹/ÔJˆÔÜ& t§}¡}¸&¿/¹/ÔJˆÖr-   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r$   r;   Úposition_embeddingsrÅ   NÚpast_key_valuesr&   )r2   r<   rK   r   )r(   r)   s   "€r*   r+   rÜ   ä   sw   ø€ ÷ %$ñ %$á—|‘|ð%$ñ #¡5§<¡<±·±Ð#=Õ>ð%$ñ Ÿ™ tÕ+ð	%$ñ
  �ð%$ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ%$r-   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      P                  ! V!  4      P                  ^^4      pV P                  V P                  V4      P                  ! V!  4      P                  ^^4      p	V P                  V4      P                  ! V!  P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pV! V VV	V
V3RRRV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pVV3# )NrÇ   ç        rÆ   r?   )rL   rr   rå   rà   Úviewr‡   ræ   rá   râ   r·   ÚupdaterÚ   r   Úget_interfacer]   Ú_attn_implementationrÕ   rÆ   r»   rÐ   rã   )r6   r;   rê   rÅ   rë   rw   Úinput_shapeÚhidden_shapeÚquery_statesrÑ   rÒ   r‰   rŠ   Úattention_interfacerÔ   rÓ   Úoutputs   &&&&&,           r*   rH   ÚLfm2Attention.forwardä   s’  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà×'Ñ'¨¯©°MÓ(B×(GÒ(GÈÑ(VÓW×aÑaÐbcÐefÓgˆØ×%Ñ% d§k¡k°-Ó&@×&EÒ&EÀ|Ñ&TÓU×_Ñ_Ð`aÐcdÓeˆ
Ø—{‘{ =Ó1×6Ò6¸ÑE×OÑOÐPQÐSTÓUˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—‘˜{Ó+ˆØ�|Ð#Ð#r-   )r]   rr   rß   ræ   rá   rÚ   rÌ   rã   rå   rà   rÆ   râ   r’   )
rO   rP   rQ   rR   Ú__doc__r0   rH   rS   rT   rU   rV   s   @@r*   r×   r×   Ñ   s#   ù‡ € áG÷Kó K÷%$÷ %$ò %$r-   r×   c                ób   € Ve+   V P                   pWR,          ,          P                  V4      p V # )ze
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
)r‚   r‚   N)r@   rA   )r;   rÅ   r@   s   && r*   Úapply_mask_to_padding_statesrú     s3   € ð
 Ò!Ø×#Ñ#ˆØ&¸
Õ)CÕC×GÑGÈÓNˆàÐr-   Úcausal_conv1d_updateÚcausal_conv1dc                óÀ   € V ^8„  d   QhR\         P                  R\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r$   r;   Ú
conv_stater4   r›   NÚ
activation©r2   r<   r   r1   r†   )r(   s   "r*   r+   r+     sS   € ÷ 'ñ 'Ü—<‘<ð'ä—‘ð'ô �L‰Lð'ô �,‰,˜Õ
ð	'ô
 �d•
ñ'r-   c                 óº  € V P                   w  rVpVP                   R,          p\        P                  ! W.RR7      P                  VP                  4      p	VP                  V	RRV) R13,          4       \        P                  ! W’P                  ^4      V^ VR7      p
V
RRV) R13,          p
Ve   \        V,          ! V
4      p
V
P                  V P                  4      # )é   r�   r‚   N)ÚpaddingÚgroupsr?   )
rL   r2   rˆ   rA   r@   Úcopy_r§   Úconv1dr±   r   )r;   rþ   r4   r›   rÿ   Ú_r7   Úseq_lenÚ	state_lenÚhidden_states_newÚouts   &&&&&      r*   rû   rû     sÅ   € ð ,×1Ñ1Ñ€A�GØ× Ñ  Õ$€IäŸ	š	 :Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ& q¨!¨i¨Z©[Ð'8Õ9Ô:Ü
�(Š(Ð$×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
ˆa��W�H‘IˆoÕ
€CØÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r-   Úcausal_conv1d_fnc          	      ó    € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R,          R\        R,          /# )r$   r;   r4   r›   Nrÿ   r   )r(   s   "r*   r+   r+   -  sE   € ÷ 'ñ 'Ü—<‘<ð'ä�L‰Lð'ô �,‰,˜Õ
ð'ô �d•
ñ	'r-   c                 óV  € V P                   w  rVpVP                   R,          ^,
          p\        P                  ! V P                  VP                  4      VP                  ^4      VVVR7      RRRV13,          p	Ve   \        V,          ! V	4      p	V	P                  V P                  4      # )r  )r4   r›   r  r  r‚   Nr?   )rL   r§   r  rA   r@   r±   r   )
r;   r4   r›   rÿ   rw   r  r7   r  r  r  s
   &&&&,     r*   r  r  ,  sž   € ð ,×1Ñ1Ñ€A�GØ�l‰l˜2Õ Õ"€Gä
�(Š(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôð ˆˆHˆWˆH€nõ€Cð ÒÜ�ZÖ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r-   c                   ój   a a€ ] tR tRt oV3R lV 3R llt]! R4      RV3R lR ll4       tRtVtV ;t	# )	ÚLfm2ShortConviC  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÙ   rÛ   )r(   r)   s   "€r*   r+   ÚLfm2ShortConv.__annotate__E  s   ø€ ÷ 8ñ 8áð8ñ ñ8r-   c           	     óv  <€ \         SV `  4        Wn        W n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  V P
                  VP                  V P                  V P
                  ^,
          R7      V n        \        P                  ! VP                  ^VP                  ,          V P                  R7      V n        \        P                  ! VP                  VP                  V P                  R7      V n        VP                  V,          V n        R# )r  )Úin_channelsÚout_channelsÚkernel_sizer  r›   r  rš   N)r/   r0   r]   rÚ   Úconv_L_cacheÚconv_kernel_sizeÚ	conv_biasr›   r   ÚConv1dr7   Úconvr¡   Úin_projrã   Úlayer_typesÚ
layer_typerç   s   &&&€r*   r0   ÚLfm2ShortConv.__init__E  sâ   ø€ ô
 	‰ÑÔØŒØ"ŒØ &× 3Ñ 3ˆÔØ×$Ñ$ˆŒ	ä—I’IØ×*Ñ*Ø×+Ñ+Ø×-Ñ-Ø×%Ñ%Ø—‘Ø×)Ñ)¨AÕ-ô
ˆŒ	ô —y’y ×!3Ñ!3°Q¸×9KÑ9KÕ5KÐRV×R[ÑR[Ô\ˆŒÜŸ	š	 &×"4Ñ"4°f×6HÑ6HÈtÏyÉyÔYˆŒà ×,Ñ,¨YÕ7ˆŽr-   r  c          	      ó˜   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          RS[ P                  R,          /# )r$   r;   rë   NrÅ   Úseq_idx)r2   r<   r   Ú	IntTensor)r(   r)   s   "€r*   r+   r  ^  sM   ø€ ÷ 'ñ 'á—|‘|ð'ñ  �ð'ñ Ÿ™ tÕ+ð	'ñ
 —‘ 4Õ'ñ'r-   c                óä  € VP                   ^,          p\        W4      pV P                  V4      P                  RR4      pVP	                  ^RR7      w  rxp	Wy,          pVRJ;'       d    VP                  V P                  4      p
V
'       d¨   V^8X  d¡   VP                  V P                  ,          P                  '       gt   VP                  V P                  ,          P                  ^ ,          p\        WV P                  P                  P                  ^4      V P                  P                  4      pMƒVe(   VP                  WP                  V P                   R7      p\#        WP                  P                  P                  ^4      V P                  P                  VR7      pVe   VRRV) R13,          pW�,          pVP                  RR4      P%                  4       pV P'                  V4      pV# )r  r�   N)r  )r!  r‚   r?   éþÿÿÿ)rL   rú   r  r‡   ÚchunkÚhas_previous_staterÚ   ÚlayersÚrecord_pastÚconv_statesrû   r  r4   Úsqueezer›   Úupdate_conv_stater  r  rÐ   rã   )r6   r;   rë   rÅ   r!  r  ÚBCxÚBÚCr‹   Úuse_precomputed_statesrþ   Úys   &&&&&        r*   rH   ÚLfm2ShortConv.forward]  sœ  € ð  ×%Ñ% aÕ(ˆä4°]ÓSˆØ�l‰l˜=Ó)×3Ñ3°B¸Ó;ˆØ—)‘)˜A 2�)Ó&‰ˆˆaØ�ˆà!0¸Ð!<×!sÐ!sÀ×AcÑAcÐdh×drÑdrÓAsÐç! g°¤l¸?×;QÑ;QÐRV×R`ÑR`Õ;a×;m×;mÐ;mØ(×/Ñ/°·±Õ?×KÑKÈAÕNˆJä0Ø¨4¯9©9×+;Ñ+;×+CÑ+CÀAÓ+FÈÏ	É	ÏÉó‰Mð Ò*Ø /× AÑ AØ!§>¡>ÀD×DYÑDYð !Bó !�ô -ØŸy™y×/Ñ/×7Ñ7¸Ó:¸D¿I¹I¿N¹NÐT[ôˆMð
 Ò*Ø -¨a°°W°H±I¨oÕ >�àÕˆØ�K‰K˜˜BÓ×*Ñ*Ó,ˆØ�M‰M˜!ÓˆØˆr-   )r›   r]   r  r  r  rÚ   r  rã   )NNN)
rO   rP   rQ   rR   r0   r   rH   rS   rT   rU   rV   s   @@r*   r  r  C  s2   ù‡ € ÷8ó 8ñ0 ˜FÓ#÷'ñ 'ó $÷'ð 'r-   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLfm2DecoderLayeriˆ  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÙ   rÛ   )r(   r)   s   "€r*   r+   ÚLfm2DecoderLayer.__annotate__‰  s   ø€ ÷ 
Mñ 
M™zð 
M±cñ 
Mr-   c                óx  <€ \         SV `  4        VP                  V,          R 8H  V n        V P                  '       d   \	        W4      V n        M\        W4      V n        \        V4      V n	        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )Úfull_attentionrÞ   N)r/   r0   r  Úis_attention_layerr×   Ú	self_attnr  r  r–   Úfeed_forwardr"   r7   rä   Úoperator_normÚffn_normrç   s   &&&€r*   r0   ÚLfm2DecoderLayer.__init__‰  sˆ   ø€ Ü‰ÑÔØ"(×"4Ñ"4°YÕ"?ÐCSÑ"SˆÔà×"×"Ð"Ü*¨6Ó=ˆD�Nä% fÓ8ˆDŒIÜ# F›OˆÔÜ(¨×);Ñ);ÀÇÁÔQˆÔÜ# F×$6Ñ$6¸F¿O¹OÔLˆŽr-   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  /# )r$   r;   rê   NrÅ   rŒ   rë   r&   )r2   r<   rK   Ú
LongTensorr   )r(   r)   s   "€r*   r+   r5  •  s~   ø€ ÷ ñ á—|‘|ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ÿ™ tÕ+ð	ñ
 ×&Ñ&¨Õ-ðñ  �ðñ 
�‰ñr-   c                óL  € TpV P                   '       d/   V P                  ! RR V P                  V4      RVRVRVRV/VB w  rM3V P                  V P                  V4      VVVP	                  R4      R7      pW,           pWP                  V P                  V4      4      ,           pV# )r;   rê   rÅ   rŒ   rë   r!  )r;   rë   rÅ   r!  © )r8  r9  r;  r  Úgetr:  r<  )	r6   r;   rê   rÅ   rŒ   rë   rw   Úresidualr  s	   &&&&&&,  r*   rH   ÚLfm2DecoderLayer.forward•  sÌ   € ð !ˆØ×"×"Ð"Ø#Ÿ~š~ñ  Ø"×0Ñ0°Ó?ð à$7ð ð  .ð ð *ð	 ð
 !0ð ð ñ ÑˆM˜1ð !ŸI™IØ"×0Ñ0°Ó?Ø /Ø-ØŸ
™
 9Ó-ð	 &ó ˆMð &Õ0ˆØ%×(9Ñ(9¸$¿-¹-ÈÓ:VÓ(WÕWˆàÐr-   )r  r:  r<  r8  r;  r9  )NNNNrª   rV   s   @@r*   r3  r3  ˆ  s    ù‡ € ÷
Mó 
M÷÷ ò r-   r3  c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚLfm2PreTrainedModeliµ  ÚmodelTr3  rë   r;   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r\   r   )r(   r)   s   "€r*   r+   Ú Lfm2PreTrainedModel.__annotate__µ  s   ø‡ ‚ áÑò r-   rA  N)rO   rP   rQ   rR   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr3  r×   Ú_can_record_outputsÚ__annotate_func__rS   rT   )r)   s   @r*   rF  rF  µ  s\   ø‡ € ð  ÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ)Ø�mðÐ÷ ƒ r-   rF  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	Lfm2ModeliÈ  c                ó    <€ V ^8„  d   QhRS[ /# r\   r   )r(   r)   s   "€r*   r+   ÚLfm2Model.__annotate__Ê  s   ø€ ÷ ñ ™zñ r-   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VR 7      V n        RV n        \#        VP                  VP$                  R7      V n        V P)                  4        R# u upi )©r]   FrÞ   N)r/   r0   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr7   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr3  r'  rX   Ú
rotary_embÚgradient_checkpointingr"   rä   Úembedding_normÚ	post_initrç   s   && €r*   r0   ÚLfm2Model.__init__Ê  sÉ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô .°VÔ<ˆŒØ&+ˆÔ#Ü)¨&×*<Ñ*<À&Ç/Á/ÔRˆÔð 	�‰Öùò cs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r$   Ú	input_idsNrÅ   rŒ   rë   Úinputs_embedsÚ	use_cacherw   r&   )	r2   r?  r<   r   ÚFloatTensorÚboolr   r   r   )r(   r)   s   "€r*   r+   rY  Ý  s“   ø€ ÷ 8
ñ 8
á×#Ñ# dÕ*ð8
ñ Ÿ™ tÕ+ð8
ñ ×&Ñ&¨Õ-ð	8
ñ
  �ð8
ñ ×(Ñ(¨4Õ/ð8
ñ ˜$•;ð8
ñ Ñ+Õ,ð8
ñ 
!ñ8
r-   c                ó4  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g/   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B /p	TpV P                  W³R7      p\        V P                   R V P                  P"                   4       F7  w  rÞV! V3RW�P                  P$                  V,          ,          RVRVRV/VB pK9  	  V P'                  V4      p\)        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr[  )rY   r]   rk  rÅ   rë   rŒ   r7  r  )rŒ   rê   )Úlast_hidden_staterë   rA  )Ú
ValueErrorr`  r   r]   Úget_seq_lengthr2   rv   rL   rY   r±   r„   Údictr   r   rd  Ú	enumerater'  rc  r  rf  r   )r6   rj  rÅ   rŒ   rë   rk  rl  rw   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr;   rê   ÚiÚdecoder_layers   &&&&&&&,       r*   rH   ÚLfm2Model.forwardÚ  s¹  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØÔ7ÑF¸+ÑFð#Ðð
 &ˆØ"Ÿo™o¨m˜oÓWÐô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð ñŠMñ !Xð ×+Ñ+¨MÓ:ˆä&Ø+Ø+ô
ð 	
r-   )r`  rf  re  r'  r]  rd  r^  )NNNNNN)rO   rP   rQ   rR   r0   r   r   r   rH   rS   rT   rU   rV   s   @@r*   rW  rW  È  s=   ù‡ € ÷ó ð   ØØ÷8
ñ 8
ó ó ó  ÷8
ð 8
r-   rW  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚLfm2ForCausalLMi  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr;   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Frš   N)
r/   r0   rW  rG  r^  r   r¡   r7   r}  rg  )r6   r]   r8   s   &&€r*   r0   ÚLfm2ForCausalLM.__init__  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör-   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r$   rj  NrÅ   rŒ   rë   rk  Úlabelsrl  Úlogits_to_keeprw   r&   )
r2   r?  r<   r   rm  rn  rž   r   r   r   )r(   r)   s   "€r*   r+   ÚLfm2ForCausalLM.__annotate__*  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r-   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ai  
Example:

```python
>>> from transformers import AutoTokenizer, Lfm2ForCausalLM

>>> model = Lfm2ForCausalLM.from_pretrained("meta-lfm2/Lfm2-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-lfm2/Lfm2-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rj  rÅ   rŒ   rë   rk  rl  Nr‚   r  r„  r^  )Úlossr  rë   r;   rH  rA  )rG  rp  r„   rž   Úslicer}  Úloss_functionr]   r^  r   rë   r;   rH  )r6   rj  rÅ   rŒ   rë   rk  r„  rl  r…  rw   Úoutputsr;   Úslice_indicesr  rˆ  s   &&&&&&&&&,     r*   rH   ÚLfm2ForCausalLM.forward(  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   )r}  rG  r^  )NNNNNNNr   )rO   rP   rQ   rR   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr0   r   r   rH   rS   rT   rU   rV   s   @@r*   r|  r|    sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r-   r|  )r  )rí   )NN)r|  rW  rF  )CÚcollections.abcr   r2   Útorch.nn.functionalr   rÎ   r§   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_lfm2r   rÈ   r"   rX   r–   r®   r·   r¿   rÕ   r×   rú   rû   r  r  r3  rF  rW  r|  Ú__all__rA  r-   r*   Ú<module>r¥     sÅ  ðõ( %ã ß Ð Ý å !ß .Ý )ß tÑ tÝ =ß PÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ñ ˜YÓ'ôJ�"—)‘)ó Jó (ðJô(3<˜"Ÿ)™)ô 3<ôl8ˆb�i‰iô 8ò((ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ô7$�B—I‘Ió 7$ó +ð7$òt	ñ (Ð(>ÀÓPö'ó Qð'ñ& (Ð(:¸OÓLö'ó Mð'ñ, Ð*Ð,<Ð=Ó>ôA�B—I‘Ió Aó ?ðAôH*Ð1ô *ðZ ô˜/ó ó ðð$ ôL
Ð#ó L
ó ðL
ð^ ôG
Ð)¨?ó G
ó ðG
òT B‚r-   