+
    œ°³j©P  ã                   óˆ  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHtHt ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHtHt ^RIHtHt ^RIHt ^RI H!t!H"t" ^RI#H$t$ ^RI%H&t&H't'H(t( ^RI)H*t* ^RI+H,t,  ! R R]PZ                  4      t.R t/]! R4      R+R l4       t0R R lt1R,R R llt2]! ]04       ! R R]PZ                  4      4       t3 ! R R]PZ                  4      t4 ! R R ]4      t5]" ! R! R"]4      4       t6]" ! R# R$]64      4       t7]" ! R% R&]6]4      4       t8 ! R' R(]]64      t9 ! R) R*]]64      t:. R-Ot;R# ).é    )ÚCallableN)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú	PhiConfigc                   óÎ   a a€ ] tR t^!t o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚPhiRotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/phi/modeling_phi.pyÚ__annotate__ÚPhiRotaryEmbedding.__annotate__#   s   ø€ ÷ Oñ O™yñ Oó    c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr%   Úrope_parametersr-   Úcompute_default_rope_parametersr   Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr%   r    Úrope_init_fnr:   Ú	__class__s   &&&  €r(   r1   ÚPhiRotaryEmbedding.__init__"   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr+   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r$   r%   Úreturn)r   ÚtupleÚtorchÚTensorÚfloat)r&   r'   s   "€r(   r)   r*   5   s.   ø€ ÷ 5ñ 5±	ð 5ÑUZÑ[`×[gÑ[gÑinÐ[nÕUoñ 5r+   c           	     óŠ  € V P                   R,          pV P                   P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        WT,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimN©Údtype)
r5   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintrD   ÚarangerF   Úto)	r%   r    ÚkwargsÚbaserI   rJ   ÚdimÚattention_factorr:   s	   &&,      r(   r6   Ú2PhiRotaryEmbedding.compute_default_rope_parameters3   s¦   € ð ×%Ñ% lÕ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rL   r    ÚmpsÚcpuF)Údevice_typeÚenabled©rV   rK   )NºNNNNéÿÿÿÿ)r_   Nr_   )r:   ÚexpandÚshaperS   rD   rF   r    Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr7   ÚsinrL   )
r=   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr\   ÚfreqsÚembrh   ri   s
   &&&       r(   ÚforwardÚPhiRotaryEmbedding.forwardI   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r7   r%   r:   r3   r<   r4   r-   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r1   Ústaticmethodr6   rD   Úno_gradr   rp   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r?   r'   s   @@r(   r   r   !   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r+   r   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr^   r`   )rb   rD   rg   )rj   Úx1Úx2s   &  r(   Úrotate_halfr€   \   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r+   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer€   )ÚqÚkrh   ri   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r(   Úapply_rotary_pos_embr‰   c   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr+   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r$   Úhidden_statesÚn_reprB   )rD   rE   rQ   )r&   s   "r(   r)   r)   }   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur+   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r_   r_   Nr_   r_   )rb   ra   Úreshape)r‹   rŒ   ÚbatchÚnum_key_value_headsÚslenrJ   s   &&    r(   Ú	repeat_kvr’   }   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr+   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r$   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrT   )r8   ÚModulerD   rE   rF   r   r   )r&   s   "r(   r)   r)   ‰   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r+   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r$   )rV   rL   )ÚpÚtrainingr`   )r’   Únum_key_value_groupsrD   Úmatmulrf   r8   Ú
functionalÚsoftmaxÚfloat32rS   rL   rš   rž   Ú
contiguous)r”   r•   r–   r—   r˜   r™   rš   rT   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r(   Úeager_attention_forwardr©   ‰   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                   óX   a a€ ] tR t^¢t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚPhiAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r$   r%   Ú	layer_idx©r   rQ   )r&   r'   s   "€r(   r)   ÚPhiAttention.__annotate__¦   s   ø€ ÷ ñ ™yð ±Sñ r+   c                ó  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP
                  VP                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        \'        V P                  VP(                  R,          ,          4      V n        VP,                  V n        V P,                  '       d‰   \        P.                  ! VP
                  VP                  ,          VP0                  RR7      V n        \        P.                  ! VP
                  VP                  ,          VP0                  RR7      V n        R# R# )rJ   T©ÚbiasrI   )ÚepsÚelementwise_affineNg      à¿)r0   r1   r%   r®   rN   rO   rP   rJ   r�   rŸ   r™   Úattention_dropoutÚ	is_causalr8   ÚLinearÚq_projÚk_projÚv_projÚdenserQ   r5   Úrotary_ndimsÚqk_layernormÚ	LayerNormÚlayer_norm_epsÚq_layernormÚk_layernorm©r=   r%   r®   r?   s   &&&€r(   r1   ÚPhiAttention.__init__¦   sË  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐeiÔjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐeiÔjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐeiÔjˆŒÜ—Y’Y˜v×9Ñ9¸D¿M¹MÕIÈ6×K]ÑK]ÐdhÔiˆŒ
Ü §¡°×0FÑ0FÐG^Õ0_Õ _Ó`ˆÔØ"×/Ñ/ˆÔØ××ÐÜ!Ÿ|š|Ø×"Ñ" f×&@Ñ&@Õ@Àf×F[ÑF[Ðptô ˆDÔô  "Ÿ|š|Ø×"Ñ" f×&@Ñ&@Õ@Àf×F[ÑF[Ðptô ˆDÖñ	 r+   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r$   r‹   Úposition_embeddingsr˜   NÚpast_key_valuesrB   )rD   rE   rC   r   )r&   r'   s   "€r(   r)   r°   ½   sw   ø€ ÷ 8)ñ 8)á—|‘|ð8)ñ #¡5§<¡<±·±Ð#=Õ>ð8)ñ Ÿ™ tÕ+ð	8)ñ
  �ð8)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ8)r+   c                ó˜  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
V P                  '       d#   V P                  V4      pV P                  V	4      p	Vw  r¼VRR V P                  13,          VRV P                  R 13,          ríV	RR V P                  13,          V	RV P                  R 13,          pp\        WßW¼4      w  rß\        P                  ! WÞ3RR7      p\        P                  ! VV3RR7      p	Ve   VP                  WšV P                  4      w  rš\         P"                  ! V P$                  P&                  \(        4      pT! V VV	V
V3RV P*                  '       g   RMV P,                  RV P.                  /VB w  ppVP0                  ! . VORN5!  P3                  4       pV P5                  V4      pVV3# )N.r^   rš   ç        r™   r`   )rb   rJ   r¹   Úviewrf   rº   r»   r¾   rÁ   rÂ   r½   r‰   rD   rg   Úupdater®   r   Úget_interfacer%   Ú_attn_implementationr©   rž   r¶   r™   rŽ   r¤   r¼   )r=   r‹   rÆ   r˜   rÇ   rT   Úinput_shapeÚhidden_shapeÚquery_statesr¥   r¦   rh   ri   Ú	query_rotÚ
query_passÚkey_rotÚkey_passÚattention_interfacer¨   r§   s   &&&&&,              r(   rp   ÚPhiAttention.forward½   sP  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà××ÐØ×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆð ˜Ð1 × 1Ñ 1Ð1Ð1Õ2Ø˜˜d×/Ñ/Ñ1Ð1Õ2ð ð
 �sÐ/˜d×/Ñ/Ð/Ð/Õ0Ø�s˜D×-Ñ-Ñ/Ð/Õ0ð ˆô
 2°)ÀcÓOÑˆ	ô —y’y )Ð!8¸bÔAˆÜ—Y’Y ¨Ð2¸Ô;ˆ
àÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—j‘j Ó-ˆØ˜LÐ(Ð(r+   )r¶   r%   r¼   rJ   r·   rÂ   rº   r®   rŸ   rÁ   r¹   r¾   r½   r™   r»   rr   )
rs   rt   ru   rv   Ú__doc__r1   rp   ry   rz   r{   r|   s   @@r(   r«   r«   ¢   s!   ù‡ € áG÷ó ÷.8)÷ 8)ò 8)r+   r«   c                   óD   a a€ ] tR t^øt oV 3R ltV3R lR ltRtVtV ;t# )ÚPhiMLPc                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # rr   )r0   r1   r%   r   Ú
hidden_actÚactivation_fnr8   r¸   rO   Úintermediate_sizeÚfc1Úfc2©r=   r%   r?   s   &&€r(   r1   ÚPhiMLP.__init__ù   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r$   r‹   rB   )rD   rE   )r&   r'   s   "€r(   r)   ÚPhiMLP.__annotate__   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r+   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rr   )rÞ   rÜ   rß   )r=   r‹   s   &&r(   rp   ÚPhiMLP.forward   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr+   )rÜ   r%   rÞ   rß   ©	rs   rt   ru   rv   r1   rp   ry   rz   r{   r|   s   @@r(   rÙ   rÙ   ø   s   ù‡ € õK÷÷ ð r+   rÙ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhiDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r­   r¯   )r&   r'   s   "€r(   r)   ÚPhiDecoderLayer.__annotate__  s   ø€ ÷ <ñ <™yð <±Sñ <r+   c                ó  <€ \         SV `  4        \        WR 7      V n        \	        V4      V n        \        P                  ! VP                  VP                  R7      V n
        \        P                  ! VP                  4      V n        R# ))r®   ©r´   N)r0   r1   r«   Ú	self_attnrÙ   Úmlpr8   r¿   rO   rÀ   Úinput_layernormÚDropoutÚresid_pdropÚresid_dropoutrÃ   s   &&&€r(   r1   ÚPhiDecoderLayer.__init__  s[   ø€ Ü‰ÑÔÜ% fÔBˆŒÜ˜&“>ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜŸZšZ¨×(:Ñ(:Ó;ˆÖr+   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r$   r‹   r˜   Nrk   rÇ   Ú	use_cacherÆ   rT   rB   )rD   rE   Ú
LongTensorr   ÚboolrC   r   r   )r&   r'   s   "€r(   r)   rê     sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr+   c                óì   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  ršV P                  V	4      p	V P                  V P                  V4      4      pW›,           V,           pV# )r‹   r˜   rk   rÇ   rõ   rÆ   © )rï   rí   rò   rî   )r=   r‹   r˜   rk   rÇ   rõ   rÆ   rT   ÚresidualÚattn_outputsÚ_Úfeed_forward_hidden_statess   &&&&&&&,    r(   rp   ÚPhiDecoderLayer.forward  sª   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ.š.ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
‰ˆð ×)Ñ)¨,Ó7ˆà%)×%7Ñ%7¸¿¹ÀÓ8OÓ%PÐ"Ø$ÕAÀHÕLˆàÐr+   )rï   rî   rò   rí   )NNNFNræ   r|   s   @@r(   rè   rè     s   ù‡ € ÷<ó <÷÷ ò r+   rè   c                   ó`   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tV 3R ltR	tV tR
# )ÚPhiPreTrainedModeli.  ÚmodelTrè   rÇ   r‹   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r#   r   )r&   r'   s   "€r(   r)   ÚPhiPreTrainedModel.__annotate__.  s   ø‡ ‚ áÑò r+   rù   N)rs   rt   ru   rv   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrè   r«   Ú_can_record_outputsÚ__annotate_func__ry   rz   )r'   s   @r(   r   r   .  s[   ø‡ € ð  ÐØ&*Ð#Ø*Ð+ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà˜Ø�lðÐ÷ ƒ r+   r   c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚPhiModeliA  c                ó    <€ V ^8„  d   QhRS[ /# r#   r   )r&   r'   s   "€r(   r)   ÚPhiModel.__annotate__C  s   ø€ ÷ ñ ™yñ r+   c           	     ót  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VR 7      V n        RV n        \
        P"                  ! VP$                  4      V n        \
        P(                  ! VP                  VP*                  R7      V n        V P/                  4        R# u upi )©r%   Frì   N)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer8   Ú	EmbeddingrO   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrè   Úlayersr   Ú
rotary_embÚgradient_checkpointingrð   Ú
embd_pdropÚembed_dropoutr¿   rÀ   Úfinal_layernormÚ	post_initrÃ   s   && €r(   r1   ÚPhiModel.__init__C  så   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÑA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô -°FÔ;ˆŒØ&+ˆÔ#ÜŸZšZ¨×(9Ñ(9Ó:ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔð 	�‰Öùò bs   ÂD5c                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r$   Ú	input_idsNr˜   rk   rÇ   Úinputs_embedsrõ   rT   rB   )	rD   rö   rE   r   ÚFloatTensorr÷   r   r   r   )r&   r'   s   "€r(   r)   r  W  s“   ø€ ÷ 4
ñ 4
á×#Ñ# dÕ*ð4
ñ Ÿ™ tÕ+ð4
ñ ×&Ñ&¨Õ-ð	4
ñ
  �ð4
ñ ×(Ñ(¨4Õ/ð4
ñ ˜$•;ð4
ñ Ñ+Õ,ð4
ñ 
!ñ4
r+   c                óª  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      p	V P                  V4      pTp
V P                  W£R7      pV P                  R V P                  P                    F  pV! V
3RV	RVRVR	VR
V/VB p
K  	  V P                  V
4      p
\!        V
VR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr  )r    )r%   r(  r˜   rÇ   rk   )rk   r˜   rk   rÇ   rõ   rÆ   )Úlast_hidden_staterÇ   )Ú
ValueErrorr  r   r%   Úget_seq_lengthrD   rR   rb   r    rƒ   r
   r"  r  r  r  r#  r   )r=   r'  r˜   rk   rÇ   r(  rõ   rT   Úpast_seen_tokensÚcausal_maskr‹   rÆ   Údecoder_layers   &&&&&&&,     r(   rp   ÚPhiModel.forwardT  sw  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð ×*Ñ*¨=Ó9ˆØ%ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ô
ð 	
r+   )r"  r  r#  r   r  r  r  r  )NNNNNN)rs   rt   ru   rv   r1   r   r   r   rp   ry   rz   r{   r|   s   @@r(   r  r  A  s=   ù‡ € ÷ó ð"  ØØ÷4
ñ 4
ó ó ó  ÷4
ð 4
r+   r  c                   ó„   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRtVtV ;t# )ÚPhiForCausalLMiŽ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr‹   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Tr²   N)
r0   r1   r  r  r  r8   r¸   rO   r4  r$  rà   s   &&€r(   r1   ÚPhiForCausalLM.__init__•  sU   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒð 	�‰Ör+   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r$   r'  Nr˜   rk   rÇ   r(  Úlabelsrõ   Úlogits_to_keeprT   rB   )
rD   rö   rE   r   r)  r÷   rQ   r   r   r   )r&   r'   s   "€r(   r)   ÚPhiForCausalLM.__annotate__   s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r+   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ac  
Example:

```python
>>> from transformers import AutoTokenizer, PhiForCausalLM

>>> model = PhiForCausalLM.from_pretrained("meta-phi/Phi-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-phi/Phi-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r'  r˜   rk   rÇ   r(  rõ   Nr_   r6  r;  r  )Úlossr6  rÇ   r‹   r  rù   )r  r+  rc   rQ   Úslicer4  Úloss_functionr%   r  r   rÇ   r‹   r  )r=   r'  r˜   rk   rÇ   r(  r;  rõ   r<  rT   Úoutputsr‹   Úslice_indicesr6  r?  s   &&&&&&&&&,     r(   rp   ÚPhiForCausalLM.forwardž  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )r4  r  r  )NNNNNNNr   )rs   rt   ru   rv   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr1   r   r   rp   ry   rz   r{   r|   s   @@r(   r3  r3  Ž  sf   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ÷6
ð 6
r+   r3  c                   ó   € ] tR tRtRtR# )ÚPhiForSequenceClassificationiÙ  rù   N©rs   rt   ru   rv   ry   rù   r+   r(   rJ  rJ  Ù  ó   † Ûr+   rJ  c                   ó   € ] tR tRtRtR# )ÚPhiForTokenClassificationiÝ  rù   NrK  rù   r+   r(   rN  rN  Ý  rL  r+   rN  )é   )rÉ   )r   r  r3  rJ  rN  )<Úcollections.abcr   rD   Útorch.nnr8   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr   r	   Úmasking_utilsr
   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_phir   r›   r   r€   r‰   r’   r©   r«   rÙ   rè   r   r  r3  rJ  rN  Ú__all__rù   r+   r(   Ú<module>rb     sP  ðõ %ã Ý å !ß .Ý )ß LÝ /÷ñ ÷
 Pß Kß FÝ &ß 7Ý 0ß YÑ YÝ 5Ý (ô8<˜Ÿ™ô 8<òv(ñ Ð-Ó.óó /ðõ2	U÷%ñ2 Ð)Ó*ôR)�2—9‘9ó R)ó +ðR)ôjˆR�Y‰Yô ô$Ð0ô $ðN ô˜ó ó ðð$ ôI
Ð!ó I
ó ðI
ðX ôG
Ð'¨ó G
ó ðG
ôT	Ð#CÐEWô 	ô	Ð =Ð?Qô 	ò‚r+   