+
    œ°³jn  ã                   óJ  € ^ RI Ht ^ RIt^ RIHu Ht ^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHt ^RIHt ^RIHtHt ^RIH t H!t! ^RI"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t* ^RI+H,t, ^RI-H.t.H/t/ ^RI0H1t1 ^RI2H3t3 ]! R4       ! R R]Ph                  4      4       t5 ! R R]Ph                  4      t6R t7]! R4      R3R l4       t8R R lt9R4R R  llt:]! ]84       ! R! R"]Ph                  4      4       t; ! R# R$]Ph                  4      t< ! R% R&]Ph                  4      t=] ! R' R(]Ph                  4      4       t> ! R) R*]Ph                  4      t? ! R+ R,]4      t@]) ! R- R.]$4      4       tA]) ! R/ R0]A4      4       tB]) ! R1 R2]A]4      4       tC. R5OtDR# )6é    )ÚCallableN)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚDots1ConfigÚRMSNormc                   óZ   a a€ ] tR t^-t oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚDots1RMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )é   ÚepsÚreturnN)Úfloat)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/dots1/modeling_dots1.pyÚ__annotate__ÚDots1RMSNorm.__annotate__/   s   ø€ ÷ $ñ $©ð $¸$ñ $ó    c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z+
Dots1RMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s   &&&€r+   r1   ÚDots1RMSNorm.__init__/   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r%   Úhidden_statesr'   ©r3   ÚTensor)r)   r*   s   "€r+   r,   r-   7   s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r.   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r%   T)Úkeepdiméÿÿÿÿ)	ÚdtypeÚtor3   Úfloat32ÚpowÚmeanÚrsqrtr6   r5   )r7   r=   Úinput_dtypeÚvariances   &&  r+   ÚforwardÚDots1RMSNorm.forward7   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r.   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)Útupler5   Úshaper6   )r7   s   &r+   Ú
extra_reprÚDots1RMSNorm.extra_repr>   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )r6   r5   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r1   rK   rP   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r9   r*   s   @@r+   r#   r#   -   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr.   r#   c                   óÎ   a a€ ] tR t^Bt o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚDots1RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r%   Úconfigr   )r)   r*   s   "€r+   r,   Ú!Dots1RotaryEmbedding.__annotate__D   s   ø€ ÷ Oñ O™{ñ Or.   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r0   r1   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr`   Úrope_parametersrc   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r7   r`   r\   Úrope_init_fnrm   r9   s   &&&  €r+   r1   ÚDots1RotaryEmbedding.__init__C   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr.   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r%   r`   r'   )r    rN   r3   r?   r(   )r)   r*   s   "€r+   r,   ra   V   s.   ø€ ÷ 5ñ 5±ð 5ÑW\Ñ]b×]iÑ]iÑkpÐ]pÕWqñ 5r.   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©rC   )ri   Úgetattrr8   Únum_attention_headsr3   Úaranger(   rD   )r`   r\   ÚkwargsÚbaseÚdimÚattention_factorrm   s   &&,    r+   rj   Ú4Dots1RotaryEmbedding.compute_default_rope_parametersT   s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r.   c                ó”  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      P	                  VP
                  4      pVR	,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! Wf3RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP	                  VP                   R7      X	P	                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabled©r|   rv   )NºNNNNrB   )r…   Nr…   )rm   r(   ÚexpandrO   rD   r\   Ú
isinstanceÚtypeÚstrr   Ú	transposer3   ÚcatÚcosrk   ÚsinrC   )
r7   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr‚   ÚfreqsÚembrŒ   r�   s
   &&&       r+   rK   ÚDots1RotaryEmbedding.forwardh   sE  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã&BF7Æ7G	)rk   r`   rm   rg   ro   rh   rc   ©N)rR   rS   rT   rU   r   r1   Ústaticmethodrj   r3   Úno_gradr   rK   rV   rW   rX   rY   s   @@r+   r[   r[   B   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r.   r[   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr„   rB   )rO   r3   r‹   )rŽ   Úx1Úx2s   &  r+   Úrotate_halfr›   x   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r.   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer›   )ÚqÚkrŒ   r�   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r+   Úapply_rotary_pos_embr¤      sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr.   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r%   r=   Ún_repr'   )r3   r?   Úint)r)   s   "r+   r,   r,   ™   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur.   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r…   r…   Nr…   r…   )rO   r†   Úreshape)r=   r¦   ÚbatchÚnum_key_value_headsÚslenru   s   &&    r+   Ú	repeat_kvr­   ™   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr.   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r%   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrz   )r   ÚModuler3   r?   r(   r   r   )r)   s   "r+   r,   r,   ¥   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r.   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r%   )r|   rC   )ÚpÚtrainingrB   )r­   Únum_key_value_groupsr3   ÚmatmulrŠ   r   Ú
functionalÚsoftmaxrE   rD   rC   rµ   r¹   Ú
contiguous)r¯   r°   r±   r²   r³   r´   rµ   rz   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r+   Úeager_attention_forwardrÃ   ¥   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r.   c                   óX   a a€ ] tR t^¾t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚDots1Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r%   r`   Ú	layer_idx©r    r§   )r)   r*   s   "€r+   r,   ÚDots1Attention.__annotate__Â   s   ø€ ÷ hñ h™{ð h±sñ hr.   c                óè  <€ \         SV `  4        \        VR 4      '       d   VP                  V,          MRV n        Wn        W n        \        VRVP                  VP                  ,          4      V n
        VP                  VP                  ,          V n        V P                  R,          V n        VP                  V n        RV n        \         P"                  ! VP                  VP                  V P                  ,          VP$                  R7      V n        \         P"                  ! VP                  VP                  V P                  ,          VP$                  R7      V n        \         P"                  ! VP                  VP                  V P                  ,          VP$                  R7      V n        \         P"                  ! VP                  V P                  ,          VP                  VP$                  R7      V n        \/        V P                  VP0                  R7      V n        \/        V P                  VP0                  R7      V n        V P                  R8X  d   VP6                  V n        R# RV n        R# )Úlayer_typesNru   T©Úbias©r&   Úsliding_attentiong      à¿)r0   r1   ÚhasattrrÌ   Ú
layer_typer`   rÈ   rw   r8   rx   ru   r«   rº   r´   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_projr#   Úrms_norm_epsÚq_normÚk_normÚsliding_window©r7   r`   rÈ   r9   s   &&&€r+   r1   ÚDots1Attention.__init__Â   s¾  ø€ Ü‰ÑÔÜ;BÀ6È=×;YÒ;Y˜&×,Ñ,¨YÖ7Ð_cˆŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô # 4§=¡=°f×6IÑ6IÔJˆŒÜ" 4§=¡=°f×6IÑ6IÔJˆŒØ7;·±ÐJ]Ô7]˜f×3Ñ3ˆÖÐcgˆÖr.   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r%   r=   Úposition_embeddingsr³   NÚpast_key_valuesrz   r'   )r3   r?   rN   r   r   r   )r)   r*   s   "€r+   r,   rÊ   Ý   s‡   ø€ ÷ ')ñ ')á—|‘|ð')ñ #¡5§<¡<±·±Ð#=Õ>ð')ñ Ÿ™ tÕ+ð	')ñ
  �ð')ñ Ñ-Õ.ð')ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ')r.   c           	     ód  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      P	                  V4      4      P                  ^^4      pV P                  V P                  V4      P	                  V4      4      P                  ^^4      p	V P                  V4      P	                  V4      P                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \         4      pT! V VV	V
V3RV P"                  '       g   RMV P$                  RV P&                  RV P(                  /VB w  rïVP*                  ! . VORN5!  P-                  4       pV P/                  V4      pWï3# )Nrµ   ç        r´   rÞ   rB   )rO   ru   rÜ   r×   ÚviewrŠ   rÝ   rØ   rÙ   r¤   ÚupdaterÈ   r   Úget_interfacer`   Ú_attn_implementationrÃ   r¹   rÓ   r´   rÞ   r©   r¾   rÚ   )r7   r=   râ   r³   rã   rz   Úinput_shapeÚhidden_shapeÚquery_statesr¿   rÀ   rŒ   r�   Úattention_interfacerÂ   rÁ   s   &&&&&,          r+   rK   ÚDots1Attention.forwardÝ   s«  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r.   )rÓ   r`   ru   rÔ   rÝ   rØ   rÈ   rÒ   rº   rÚ   rÜ   r×   r´   rÞ   rÙ   r•   ©
rR   rS   rT   rU   Ú__doc__r1   rK   rV   rW   rX   rY   s   @@r+   rÅ   rÅ   ¾   s#   ù‡ € áG÷hó h÷6')÷ ')ò ')r.   rÅ   c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚDots1MLPi  c                óä  <€ \         SV `  4        Wn        VP                  V n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        \        VP                  ,          V n        R # )NFrÍ   )r0   r1   r`   r8   Úintermediate_sizer   rÕ   Ú	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r7   r`   rô   r9   s   &&&€r+   r1   ÚDots1MLP.__init__  s¬   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ=NÒ=V ×!9Ò!9Ð\mˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r•   )r÷   rù   rõ   rö   )r7   rŽ   r÷   s   && r+   rK   ÚDots1MLP.forward  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr.   )rù   r`   r÷   rõ   r8   rô   rö   r•   ©	rR   rS   rT   rU   r1   rK   rV   rW   rX   rY   s   @@r+   rò   rò     s   ù‡ € ÷0÷ò r.   rò   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚDots1TopkRouteri  c                ó    <€ V ^8„  d   QhRS[ /# r_   r   )r)   r*   s   "€r+   r,   ÚDots1TopkRouter.__annotate__  s   ø€ ÷ 
Pñ 
P™{ñ 
Pr.   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        VP                  V n        VP                  V n        VP                   V n        VP"                  V n        \        P$                  ! \        P                  ! V P
                  4      4      V n        R # r•   )r0   r1   Únum_experts_per_tokÚtop_kÚnum_local_expertsÚnum_expertsr8   Ú
hidden_dimr   r2   r3   Úzerosr5   Úrouted_scaling_factorÚn_groupÚ	num_groupÚ
topk_groupÚnorm_topk_probrl   Úe_score_correction_bias©r7   r`   r9   s   &&€r+   r1   ÚDots1TopkRouter.__init__  s¯   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒØ%+×%AÑ%AˆÔ"ØŸ™ˆŒØ ×+Ñ+ˆŒØ$×3Ñ3ˆÔÜ')§y¢y´·²¸T×=MÑ=MÓ1NÓ'OˆÖ$r.   c                ó¼  € VP                  RV P                  4      p\        P                  ! VP	                  \
        P                  4      V P                  P	                  \
        P                  4      4      pVP                  4       pW0P                  ,           pVP                  RV P                  V P                  V P                  ,          4      P                  ^RR7      ^ ,          P                  RR7      p\
        P                  ! WPP                  RRR7      ^,          p\
        P                  ! V4      pVP!                  ^V^4       VP#                  R4      P%                  RV P                  V P                  V P                  ,          4      P'                  RV P                  4      pVP)                  VP+                  4       ( \-        R4      4      p\
        P                  ! W@P.                  RRR7      ^,          p	VP1                  ^V	4      p
V P2                  '       d#   V
P                  RRR7      R,           pW«,          p
W P4                  ,          p
W*V	3# )	é   r„   F)r    r|   Úsortedz-infT)r|   rA   g#B’¡œÇ;rB   )ræ   r  ÚFÚlinearrˆ   r3   rE   r5   Úsigmoidr  r  r  ÚtopkÚsumr  Ú
zeros_likeÚscatter_rž   r†   r©   Úmasked_fillÚboolr(   r  Úgatherr  r	  )r7   r=   Úrouter_logitsÚscoresÚscores_for_choiceÚgroup_scoresÚ	group_idxÚ
group_maskÚ
score_maskÚtopk_indicesÚtopk_weightsÚdenominators   &&          r+   rK   ÚDots1TopkRouter.forward$  sÒ  € Ø%×*Ñ*¨2¨t¯©Ó?ˆÜŸš ×!3Ñ!3´E·M±MÓ!BÀDÇKÁK×DTÑDTÔUZ×UbÑUbÓDcÓdˆØ×&Ñ&Ó(ˆØ"×%AÑ%AÕAÐà×"Ñ" 2 t§~¡~°t×7GÑ7GÈ4Ï>É>Õ7YÓZß‰T�!˜ˆT‹_˜Qõ ç‰S�RˆS‹[ð 	ô
 —J’J˜|¯©ÀBÈuÔUÐVWÕXˆ	Ü×%Ò% lÓ3ˆ
Ø×Ñ˜A˜y¨!Ô,à× Ñ  Ó$ß‰V�B˜Ÿ™¨×(8Ñ(8¸D¿N¹NÕ(JÓKß‰W�R˜×)Ñ)Ó*ð 	ð
 .×9Ñ9¸:¿?¹?Ó;LÐ:LÌeÐTZËmÓ\ÐÜ—z’zÐ"3·z±zÀrÐRWÔXÐYZÕ[ˆØ—}‘} Q¨Ó5ˆØ××ÐØ&×*Ñ*¨r¸4Ð*Ó@À5ÕHˆKØÕ'ˆLØ#×&@Ñ&@Õ@ˆØ¨LÐ8Ð8r.   )	r  r  r  r  r  r	  r  r  r5   rý   rY   s   @@r+   rÿ   rÿ     s   ù‡ € ÷
Pó 
P÷9ò 9r.   rÿ   c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚDots1Expertsi@  z2Collection of expert weights stored as 3D tensors.c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        R# )r%   N)r0   r1   r  r  r8   r  Úmoe_intermediate_sizeÚintermediate_dimr   r2   r3   ÚemptyÚgate_up_projr÷   r   rø   rù   r  s   &&€r+   r1   ÚDots1Experts.__init__D  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŽr.   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r%   r=   Útop_k_indexÚtop_k_weightsr'   r>   )r)   r*   s   "€r+   r,   ÚDots1Experts.__annotate__M  sC   ø€ ÷ #ñ #á—|‘|ð#ñ —\‘\ð#ñ —|‘|ð	#ñ
 
�‰ñ#r.   c                óÆ  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \         P                  P                  P                  W P                  R 7      pVP                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X EF  pV^ ,          pWpP                  8X  d   K  \         P                  ! XV,          4      w  r‰W,          p
\        P                  P                  W P                  V,          4      P                  ^RR7      w  r¼V P                  V4      V,          p\        P                  P                  WÐP                   V,          4      pWÓW˜R3,          ,          pVP#                  ^ W�P%                  VP&                  4      4       EK  	  V#   + '       g   i     EL'; i))Únum_classesr„   NrB   )rB   éþÿÿÿ)r3   r  r—   r   r¼   Úone_hotr  ÚpermuteÚgreaterr  ÚnonzeroÚwherer  r/  Úchunkrù   r÷   Ú
index_add_rD   rC   )r7   r=   r2  r3  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess   &&&&          r+   rK   ÚDots1Experts.forwardM  ss  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷ ô
 %ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨{¸:Õ/FÓ#GÑ ˆIØ)Õ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÕ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆDØ$(§K¡K°Ó$5¸Õ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÕPjÓ$kÐ!Ø$9È)Ð`dÐJdÕ<eÕ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9l×mñ %ð #Ð"÷# �_ˆ_ús   ´A=GÇG 	)rù   r÷   r/  r  r-  r  rï   rY   s   @@r+   r*  r*  @  s   ù‡ € á<õ0÷#÷ #ð #r.   r*  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚDots1MoEih  z2
A mixed expert module containing shared experts.
c                ó    <€ V ^8„  d   QhRS[ /# r_   r   )r)   r*   s   "€r+   r,   ÚDots1MoE.__annotate__m  s   ø€ ÷ 
ñ 
™{ñ 
r.   c                óÈ   <€ \         SV `  4        Wn        \        V4      V n        \        V4      V n        \        WP                  VP                  ,          R 7      V n
        R# ))r`   rô   N)r0   r1   r`   r*  Úexpertsrÿ   rF  rò   r,  Ún_shared_expertsÚshared_expertsr  s   &&€r+   r1   ÚDots1MoE.__init__m  sL   ø€ Ü‰ÑÔØŒÜ# FÓ+ˆŒÜ# FÓ+ˆŒ	Ü&Ø×-IÑ-IÈF×LcÑLcÕ-cô
ˆÖr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r<   r>   )r)   r*   s   "€r+   r,   rM  v  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r.   c                óþ   € TpVP                   pV P                  V4      w  rEpVP                  RVP                   R,          4      pV P                  WV4      P                  ! V!  pWP	                  V4      ,           pV# )r  rB   )rO   rF  ræ   rO  rQ  )r7   r=   Ú	residualsÚ
orig_shapeÚ_r&  r%  s   &&     r+   rK   ÚDots1MoE.forwardv  sw   € Ø!ˆ	Ø"×(Ñ(ˆ
Ø(,¯	©	°-Ó(@Ñ%ˆ˜Ø%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØŸ™ ]À,ÓO×TÒTÐV`ÑaˆØ%×(;Ñ(;¸IÓ(FÕFˆØÐr.   )r`   rO  rF  rQ  rï   rY   s   @@r+   rK  rK  h  s#   ù‡ € ñ÷
ó 
÷÷ ð r.   rK  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDots1DecoderLayeri€  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÇ   rÉ   )r)   r*   s   "€r+   r,   ÚDots1DecoderLayer.__annotate__�  s   ø€ ÷ bñ b™{ð b±sñ br.   c                ób  <€ \         SV `  4        VP                  V n        \        WR 7      V n        W!P
                  8¼  d   \        V4      V n        M\        V4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# ))r`   rÈ   rÏ   N)r0   r1   r8   rÅ   Ú	self_attnÚfirst_k_dense_replacerK  Úmlprò   r#   rÛ   Úinput_layernormÚpost_attention_layernormrß   s   &&&€r+   r1   ÚDots1DecoderLayer.__init__�  s‚   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä'¨vÔKˆŒà×4Ñ4Ô4Ü Ó'ˆD�Hä Ó'ˆDŒHä+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÖ%r.   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r%   r=   r³   Nr�   rã   Ú	use_cacherâ   rz   r'   )r3   r?   Ú
LongTensorr   r  rN   r   r   )r)   r*   s   "€r+   r,   r\  �  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr.   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r=   r³   r�   rã   re  râ   © )ra  r^  rb  r`  )
r7   r=   r³   r�   rã   re  râ   rz   ÚresidualrW  s
   &&&&&&&,  r+   rK   ÚDots1DecoderLayer.forward�  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr.   )r8   ra  r`  rb  r^  )NNNFNrý   rY   s   @@r+   rZ  rZ  €  s    ù‡ € ÷bó b÷÷ ò r.   rZ  c                   ó¤   a a€ ] tR tRt oRtRtR.tR.tRtRt	Rt
RtRtR]R]/tR.tR	t]P&                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚDots1PreTrainedModeli¯  ÚmodelTrZ  rã   r=   Ú
attentionsr  Nc                ó  <€ \         SV `  V4       \        V\        4      '       dZ   \        P
                  ! VP                  R V P                  P                  R7       \        P                  ! VP                  4       R# \        V\        4      '       dq   \        P
                  ! VP                  R V P                  P                  R7       \        P
                  ! VP                  R V P                  P                  R7       R# R# )rå   )rG   ÚstdN)r0   Ú_init_weightsr‡   rÿ   ÚinitÚnormal_r5   r`   Úinitializer_rangeÚzeros_r  r*  r/  r÷   )r7   r¯   r9   s   &&€r+   rq  Ú"Dots1PreTrainedModel._init_weightsÃ  sŸ   ø€ ä‰Ñ˜fÔ%Ü�fœo×.Ò.Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6SÕTÜ�KŠK˜×6Ñ6Ö7Ü˜¤×-Ò-Ü�LŠL˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�LŠL˜×)Ñ)°¸¿¹×9VÑ9V×Wñ .r.   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r_   r   )r)   r*   s   "€r+   r,   Ú!Dots1PreTrainedModel.__annotate__¯  s   ø‡ ‚ áÑò r.   rh  )rR   rS   rT   rU   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrZ  rÅ   Ú_can_record_outputsÚ_keep_in_fp32_modules_strictÚ"_keys_to_ignore_on_load_unexpectedr3   r—   rq  Ú__annotate_func__rV   rW   rX   rY   s   @@r+   rl  rl  ¯  s‡   ù‡ € ð  ÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ*Ø�nðÐð %>Ð#>Ð Ø)-Ð&à
‡]‚]ƒ_ôXó ðX÷+ … r.   rl  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú
Dots1ModeliÎ  c                ó    <€ V ^8„  d   QhRS[ /# r_   r   )r)   r*   s   "€r+   r,   ÚDots1Model.__annotate__Ð  s   ø€ ÷ ñ ™{ñ r.   c           	     óP  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        RV P(                  P*                  9   V n        V P/                  4        R# u upi )rÏ   ©r`   FrÐ   N)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr8   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrZ  Úlayersr#   rÛ   Únormr[   Ú
rotary_embÚgradient_checkpointingr`   rÌ   Úhas_sliding_layersÚ	post_initrß   s   && €r+   r1   ÚDots1Model.__init__Ð  sá   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÑCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#Ø"5¸¿¹×9PÑ9PÑ"PˆÔð 	�‰Öùò ds   ÂD#c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r%   Ú	input_idsNr³   r�   rã   Úinputs_embedsre  rz   r'   )	r3   rf  r?   r   ÚFloatTensorr  r   r   r   )r)   r*   s   "€r+   r,   r‰  ä  s“   ø€ ÷ <
ñ <
á×#Ñ# dÕ*ð<
ñ Ÿ™ tÕ+ð<
ñ ×&Ñ&¨Õ-ð	<
ñ
  �ð<
ñ ×(Ñ(¨4Õ/ð<
ñ ˜$•;ð<
ñ Ñ+Õ,ð<
ñ 
!ñ<
r.   c                ó~  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gD   RV P                  RVRVRVRV/p
R	\        R/ V
B /p	V P                  '       d   \        R/ V
B V	R
&   TpV P                  W³4      p\!        V P"                  R V P                  P$                   4       F9  w  rÞV! V3RW�P                  P&                  V,          ,          RVRVRVRV/VB pK;  	  V P)                  V4      p\+        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr‹  )r\   r`   r�  r³   rã   r�   Úfull_attentionrÐ   râ   re  )Úlast_hidden_staterã   rh  )Ú
ValueErrorr�  r   r`   Úget_seq_lengthr3   ry   rO   r\   rž   r‡   Údictr   r˜  r   r–  Ú	enumerater”  r“  rÌ   r•  r   )r7   rœ  r³   r�   rã   r�  re  rz   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr=   râ   ÚiÚdecoder_layers   &&&&&&&,       r+   rK   ÚDots1Model.forwardá  sã  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"Cð#Ðð ×&×&Ð&Ü;\Ñ;kÐ_jÑ;kÐ#Ð$7Ñ8à%ˆØ"Ÿo™o¨mÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8ðð *ð	ð
 !0ðð $ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆÜ&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r.   )r�  r—  r˜  r”  r•  r�  r–  rŽ  )NNNNNN)rR   rS   rT   rU   r1   r   r   r   rK   rV   rW   rX   rY   s   @@r+   r‡  r‡  Î  s=   ù‡ € ÷ó ð"  ØØ÷<
ñ <
ó ó ó  ÷<
ð <
r.   r‡  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚDots1ForCausalLMi#  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr=   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )FrÍ   N)
r0   r1   r‡  rm  rŽ  r   rÕ   r8   r®  r™  r  s   &&€r+   r1   ÚDots1ForCausalLM.__init__*  sU   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör.   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r%   rœ  Nr³   r�   rã   r�  Úlabelsre  Úlogits_to_keeprz   r'   )
r3   rf  r?   r   rž  r  r§   r   r   r   )r)   r*   s   "€r+   r,   ÚDots1ForCausalLM.__annotate__5  s»   ø€ ÷ ;
ñ ;
á×#Ñ# dÕ*ð;
ñ Ÿ™ tÕ+ð;
ñ ×&Ñ&¨Õ-ð	;
ñ
  �ð;
ñ ×(Ñ(¨4Õ/ð;
ñ × Ñ  4Õ'ð;
ñ ˜$•;ð;
ñ ™eŸl™lÕ*ð;
ñ Ñ+Õ,ð;
ñ 
 ñ;
r.   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, Dots1ForCausalLM

>>> model = Dots1ForCausalLM.from_pretrained("rednote-hilab/dots1.llm1.inst")
>>> tokenizer = AutoTokenizer.from_pretrained("rednote-hilab/dots1.llm1.inst")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rœ  r³   r�   rã   r�  re  Nr…   r°  rµ  rŽ  )Úlossr°  rã   r=   rn  rh  )rm  r¡  r‡   r§   Úslicer®  Úloss_functionr`   rŽ  r   rã   r=   rn  )r7   rœ  r³   r�   rã   r�  rµ  re  r¶  rz   Úoutputsr=   Úslice_indicesr°  r¹  s   &&&&&&&&&,     r+   rK   ÚDots1ForCausalLM.forward3  s   € ðH ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   )r®  rm  rŽ  )NNNNNNNr   )rR   rS   rT   rU   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr1   r   r   rK   rV   rW   rX   rY   s   @@r+   r­  r­  #  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷;
ñ ;
ó ó ÷;
ð ;
r.   r­  )r  )rå   )rl  r‡  r­  )EÚcollections.abcr   r3   Útorch.nn.functionalr   r¼   r  Ú r   rr  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_dots1r    r¶   r#   r[   r›   r¤   r­   rÃ   rÅ   rò   rÿ   r*  rK  rZ  rl  r‡  r­  Ú__all__rh  r.   r+   Ú<module>r×     s¡  ðõ( %ã ß Ð Ý å &Ý !ß .Ý )ß hÑ hß RÝ BÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý ,ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(3<˜2Ÿ9™9ô 3<òl(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ôE)�R—Y‘Yó E)ó +ðE)ôPˆr�y‰yô ô &9�b—i‘iô &9ðR ô$#�2—9‘9ó $#ó ð$#ôNˆr�y‰yô ô0,Ð2ô ,ð^ ôX˜?ó Xó ðXð< ôQ
Ð%ó Q
ó ðQ
ðh ôL
Ð+¨_ó L
ó ðL
ò^ E‚r.   