+
    œ°³jC[  ã                   ó¨  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtHt ^RI	H
t
 ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHt ^RIHtHt ^RIHtHt ^RIHtHt ^RIH t  ^RI!H"t"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t) ^RI*H+t+ ^RI,H-t-  ! R R]P\                  4      t/ ! R R]P\                  4      t0R t1R R lt2R/R R llt3R0R lt4 ! R R]P\                  4      t5]! R 4       ! R! R"]P\                  4      4       t6 ! R# R$]4      t7]# ! R% R&]4      4       t8]# ! R' R(]84      4       t9]# ! R) R*]8]
4      4       t: ! R+ R,]]84      t; ! R- R.]]84      t<. R1Ot=R# )2é    )ÚCallableN)Únn)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©Ú
Phi3Configc                   óD   a a€ ] tR t^1t oV 3R ltV3R lR ltRtVtV ;t# )ÚPhi3MLPc                ó>  <€ \         SV `  4        Wn        \        P                  ! VP
                  ^VP                  ,          RR7      V n        \        P                  ! VP                  VP
                  RR7      V n        \        VP                  ,          V n        R# )é   F©ÚbiasN)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©Úselfr(   Ú	__class__s   &&€Úg/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/phi3/modeling_phi3.pyr'   ÚPhi3MLP.__init__2   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÕ:VÐ]bÔcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Õ6ˆÖó    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r#   Úhidden_statesÚreturn)ÚtorchÚFloatTensor)ÚformatÚ__classdict__s   "€r3   Ú__annotate__ÚPhi3MLP.__annotate__:   s'   ø€ ÷ )ñ )¡U×%6Ñ%6ð )¹5×;LÑ;Lñ )r5   c                óž   € V P                  V4      pVP                  ^RR7      w  r2W P                  V4      ,          pV P                  V4      # )r#   ©Údiméÿÿÿÿ)r,   Úchunkr/   r-   )r1   r8   Ú	up_statesÚgates   &&  r3   ÚforwardÚPhi3MLP.forward:   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆØ× 2Ñ 2°4Ó 8Õ8ˆ	à�~‰~˜iÓ(Ð(r5   )r/   r(   r-   r,   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r'   rG   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r2   r=   s   @@r3   r!   r!   1   s   ù‡ € õ7÷)÷ )ð )r5   r!   c                   óÎ   a a€ ] tR t^Ct o]! RRR7      R
V3R lV 3R lll4       t]]! RRR7      R
V3R lR ll4       4       t]P                  ! 4       ]
R 4       4       tR	tVtV ;t# )ÚPhi3RotaryEmbeddingÚdevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©r#   r(   r   )r<   r=   s   "€r3   r>   Ú Phi3RotaryEmbedding.__annotate__E   s   ø€ ÷ Oñ O™zñ Or5   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚdefaultF)Ú
persistentN)r&   r'   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   Úrope_parametersrZ   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r1   r(   rT   Úrope_init_fnrd   r2   s   &&&  €r3   r'   ÚPhi3RotaryEmbedding.__init__D   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr5   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r#   r(   r9   )r   Útupler:   ÚTensorÚfloat)r<   r=   s   "€r3   r>   rX   W   s.   ø€ ÷ 5ñ 5±
ð 5ÑV[Ñ\a×\hÑ\hÑjoÐ\oÕVpñ 5r5   c           	     óŠ  € V P                   R,          pV P                   P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        WT,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimN©Údtype)
r`   ÚgetÚgetattrr*   Únum_attention_headsÚintr:   Úarangerl   Úto)	r(   rT   ÚkwargsÚbasero   rp   rB   Úattention_factorrd   s	   &&,      r3   ra   Ú3Phi3RotaryEmbedding.compute_default_rope_parametersU   s¦   € ð ×%Ñ% lÕ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r5   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)N)rr   rT   ÚmpsÚcpuF)Údevice_typeÚenabledrA   rq   )NºNNNNrC   )r‚   Nr‚   )rd   ÚexpandÚshaperx   r:   rl   rT   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrb   Úsinrr   )
r1   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr€   ÚfreqsÚembrŠ   r‹   s
   &&&       r3   rG   ÚPhi3RotaryEmbedding.forwardk   s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)rb   r(   rd   r^   rf   r_   rZ   ©N)rJ   rK   rL   rM   r   r'   Ústaticmethodra   r:   Úno_gradr   rG   rN   rO   rP   rQ   s   @@r3   rS   rS   C   su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ÷<ð <r5   rS   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..NrA   rC   )r„   r:   r‰   )rŒ   Úx1Úx2s   &  r3   Úrotate_halfr™   ~   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r5   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r#   r8   Ún_repr9   )r:   rk   rv   )r<   s   "r3   r>   r>   …   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur5   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)r‚   r‚   Nr‚   r‚   )r„   rƒ   Úreshape)r8   r›   ÚbatchÚnum_key_value_headsÚslenrp   s   &&    r3   Ú	repeat_kvr¡   …   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr5   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r#   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutry   )r   ÚModuler:   rk   rl   r   r   )r<   s   "r3   r>   r>   ‘   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r5   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r#   )rB   rr   )ÚpÚtrainingrC   )r¡   Únum_key_value_groupsr:   Úmatmulrˆ   r   Ú
functionalÚsoftmaxÚfloat32rx   rr   r©   r­   Ú
contiguous)r£   r¤   r¥   r¦   r§   r¨   r©   ry   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r3   Úeager_attention_forwardr¸   ‘   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r5   c                ó²  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜\        P                  ! Wb,          \	        V4      V,          ,           V.RR7      p
\        P                  ! W‚,          \	        V4      V,          ,           V	.RR7      pW«3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.NrA   rC   )Ú	unsqueezer„   r:   r‰   r™   )ÚqÚkrŠ   r‹   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r3   Úapply_rotary_pos_embrÅ   ª   sÆ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cà—‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ä�iŠi˜%�+¬+°eÓ*<¸sÕ*BÕCÀVÐLÐRTÔU€GÜ�iŠi˜%�+¬+°eÓ*<¸sÕ*BÕCÀVÐLÐRTÔU€GØÐÐr5   c                   ó\   a a€ ] tR t^Èt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )ÚPhi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r#   r(   Ú	layer_idxN©r   rv   )r<   r=   s   "€r3   r>   ÚPhi3Attention.__annotate__Ë   s#   ø€ ÷ Kñ K™zð K±c¸Dµjñ Kr5   c                ó¾  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        VP                  V n        V P                  R,          V n
        VP                  V n        RV n        VP                  V P                  ,          ^VP                  V P                  ,          ,          ,           p\        P                  ! VP                  V P                  ,          VP
                  RR7      V n        \        P                  ! VP
                  VRR7      V n        R# )rp   TFr$   Ng      à¿)r&   r'   r(   rÉ   rt   r*   ru   rp   rŸ   r®   r¨   Úattention_dropoutÚ	is_causalr   r)   Úo_projÚqkv_proj)r1   r(   rÉ   Úop_sizer2   s   &&& €r3   r'   ÚPhi3Attention.__init__Ë   sù   ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒà×,Ñ,¨t¯}©}Õ<¸qÀF×D^ÑD^Ðae×anÑanÕDnÕ?oÕoˆÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŒÜŸ	š	 &×"4Ñ"4°gÀEÔJˆŽr5   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r#   r8   Úposition_embeddingsr§   NÚpast_key_valuesry   r9   )r:   rk   rj   r   r   r   )r<   r=   s   "€r3   r>   rË   Ú   s—   ø€ ÷ -)ñ -)á—|‘|ð-)ñ #¡5§<¡<±·±Ð#=Õ>ð-)ñ Ÿ™ tÕ+ð	-)ñ
  �ð-)ñ Ñ-Õ.ð-)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ-)r5   c                ó*  € VP                   R R p. VORNV P                  N5pV P                  V4      pV P                  P                  V P                  ,          p	VRR V	13,          p
VRW™V P
                  V P                  ,          ,           13,          pVRW�P
                  V P                  ,          ,           R 13,          pV
P                  V4      P                  ^^4      p
VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W«WÞ4      w  r«Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                  \        4      pT! V V
VVV3RV P                  '       g   RMV P                   RV P"                  R\%        V P                  RR 4      /VB w  ppVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pVV3# )N.r©   ç        r¨   Úsliding_windowrC   )r„   rp   rÐ   r(   ru   rŸ   Úviewrˆ   rÅ   ÚupdaterÉ   r   Úget_interfaceÚ_attn_implementationr¸   r­   rÍ   r¨   rt   r�   r³   rÏ   )r1   r8   rÔ   r§   rÕ   ry   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesr´   rµ   rŠ   r‹   Úattention_interfacer·   r¶   s   &&&&&,            r3   rG   ÚPhi3Attention.forwardÚ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà�m‰m˜MÓ*ˆØ—K‘K×3Ñ3°d·m±mÕCˆ	Ø˜3 
  
˜?Õ+ˆØ˜˜i°d×6NÑ6NÐQU×Q^ÑQ^Õ6^Õ*^Ð^Ð^Õ_ˆ
Ø˜3 	×,DÑ,DÀtÇ}Á}Õ,TÕ TÑ VÐVÕWˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r5   )
rÍ   r(   rp   rÎ   rÉ   r®   rŸ   rÏ   rÐ   r¨   r“   )
rJ   rK   rL   rM   Ú__doc__r'   rG   rN   rO   rP   rQ   s   @@r3   rÇ   rÇ   È   s#   ù‡ € ÙG÷Kõ K÷-)÷ -)ò -)r5   rÇ   ÚRMSNormc                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	ÚPhi3RMSNormi
  c                ó$   <€ V ^8„  d   QhRS[ RR/# )r#   Úepsr9   N)rl   )r<   r=   s   "€r3   r>   ÚPhi3RMSNorm.__annotate__  s   ø€ ÷ $ñ $©ð $¸$ñ $r5   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z*
Phi3RMSNorm is equivalent to T5LayerNorm
N)r&   r'   r   Ú	Parameterr:   ÚonesÚweightÚvariance_epsilon)r1   r*   ré   r2   s   &&&€r3   r'   ÚPhi3RMSNorm.__init__  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör5   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r7   )r:   rk   )r<   r=   s   "€r3   r>   rê     s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r5   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r#   T)ÚkeepdimrC   )	rr   rx   r:   r²   ÚpowÚmeanÚrsqrtrï   rî   )r1   r8   Úinput_dtypeÚvariances   &&  r3   rG   ÚPhi3RMSNorm.forward  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r5   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rj   rî   r„   rï   )r1   s   &r3   Ú
extra_reprÚPhi3RMSNorm.extra_repr  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr5   )rï   rî   )g�íµ ÷Æ°>)
rJ   rK   rL   rM   r'   rG   rû   rN   rO   rP   rQ   s   @@r3   rç   rç   
  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr5   rç   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhi3DecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   r(   rÉ   rÊ   )r<   r=   s   "€r3   r>   ÚPhi3DecoderLayer.__annotate__   s   ø€ ÷ 	@ñ 	@™zð 	@±cñ 	@r5   c                óÀ  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        Wn        \        P                  ! VP                  4      V n        \        P                  ! VP                  4      V n        R# ))r(   rÉ   ©ré   N)r&   r'   r*   rÇ   Ú	self_attnr!   Úmlprç   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr(   r   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout©r1   r(   rÉ   r2   s   &&&€r3   r'   ÚPhi3DecoderLayer.__init__   sœ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ&¨fÔJˆŒÜ˜6“?ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%ØŒÜ"$§*¢*¨V×-?Ñ-?Ó"@ˆÔÜ!#§¢¨F×,>Ñ,>Ó!?ˆÖr5   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r#   r8   r§   Nr�   rÕ   Ú	use_cacherÔ   ry   r9   )r:   rk   Ú
LongTensorr   Úboolrj   r   r   )r<   r=   s   "€r3   r>   r   +  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ-Õ.ðñ 
�‰ñr5   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW€P                  V4      ,           pTpV P                  V4      pV P	                  V4      pW€P                  V4      ,           pV# )r8   r§   r�   rÕ   r  rÔ   © )r  r  r
  r  r  r  )
r1   r8   r§   r�   rÕ   r  rÔ   ry   ÚresidualÚself_attn_weightss
   &&&&&&&,  r3   rG   ÚPhi3DecoderLayer.forward+  s½   € ð !ˆØ×,Ñ,¨]Ó;ˆà+/¯>ª>ñ ,
Ø'ð,
à)ð,
ð &ð,
ð ,ð	,
ð
  ð,
ð !4ð,
ð ñ,
Ñ(ˆð !×#:Ñ#:¸=Ó#IÕIˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ ×#9Ñ#9¸-Ó#HÕHˆØÐr5   )r(   r*   r  r  r  r
  r  r  )NNNFNrI   rQ   s   @@r3   rþ   rþ     s    ù‡ € ÷	@ó 	@÷÷ ò r5   rþ   c                   ód   a € ] tR tRt o RtRtR.tR.tRtRt	Rt
RtRtR]R]/tRtV 3R	 ltR
tV tR# )ÚPhi3PreTrainedModeliJ  ÚmodelTrþ   rÕ   r8   Ú
attentionsz0.0.5c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rW   r   )r<   r=   s   "€r3   r>   Ú Phi3PreTrainedModel.__annotate__J  s   ø‡ ‚ áÑò r5   r  N)rJ   rK   rL   rM   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrþ   rÇ   Ú_can_record_outputsÚ_versionÚ__annotate_func__rN   rO   )r=   s   @r3   r  r  J  sc   ø‡ € ð  ÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐ)Ø�mðÐð €H÷# ƒ r5   r  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	Phi3Modeli^  c                ó    <€ V ^8„  d   QhRS[ /# rW   r   )r<   r=   s   "€r3   r>   ÚPhi3Model.__annotate__`  s   ø€ ÷ ñ ™zñ r5   c           	     ó  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        V P)                  4        R# u upi )r  ©r(   FN)r&   r'   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr*   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrþ   Úlayersrç   r  ÚnormrS   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s   && €r3   r'   ÚPhi3Model.__init__`  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÑBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Öùò cs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r#   Ú	input_idsNr§   r�   rÕ   Úinputs_embedsr  ry   r9   )	r:   r  rk   r   r;   r  r   r   r   )r<   r=   s   "€r3   r>   r,  s  s“   ø€ ÷ 2
ñ 2
á×#Ñ# dÕ*ð2
ñ Ÿ™ tÕ+ð2
ñ ×&Ñ&¨Õ-ð	2
ñ
  �ð2
ñ ×(Ñ(¨4Õ/ð2
ñ ˜$•;ð2
ñ Ñ+Õ,ð2
ñ 
!ñ2
r5   c                óè  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pV P                  P                  f   \        M\        p	V	! V P                  VVVVR7      p
TpV P                  W³R7      pV P                  R V P                  P                    F  pV! V3RV
RVRVR	VR
V/VB pK  	  V P!                  V4      p\#        TV'       d   VR7      # R R7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr.  )rT   )r(   r?  r§   rÕ   r�   )r�   r§   r�   rÕ   r  rÔ   )Úlast_hidden_staterÕ   )Ú
ValueErrorr3  r   r(   Úget_seq_lengthr:   rw   r„   rT   rº   rØ   r
   r   r9  r7  r6  r8  r   )r1   r>  r§   r�   rÕ   r?  r  ry   Úpast_seen_tokensÚmask_functionÚcausal_maskr8   rÔ   Údecoder_layers   &&&&&&&,      r3   rG   ÚPhi3Model.forwardp  sŒ  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÒ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð %8ðð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+ß/8˜Oô
ð 	
à>Bô
ð 	
r5   )r3  r:  r7  r8  r0  r9  r1  )NNNNNN)rJ   rK   rL   rM   r'   r   r   r   rG   rN   rO   rP   rQ   s   @@r3   r*  r*  ^  s=   ù‡ € ÷ó ð   ØØ÷2
ñ 2
ó ó ó  ÷2
ð 2
r5   r*  c                   ó”   a a€ ] tR tRt oRR/tRR/tRR.R.3/tRR/tV 3R	 lt]	]
RV3R
 lR ll4       4       tRV 3R lltRtVtV ;t# )ÚPhi3ForCausalLMi¨  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr8   ÚlogitsÚkeep_full_weightc                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# )Fr$   N)
r&   r'   r*  r  r1  r   r)   r*   rK  r;  r0   s   &&€r3   r'   ÚPhi3ForCausalLM.__init__¯  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör5   c                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )r#   r>  Nr§   r�   rÕ   r?  Úlabelsr  Úlogits_to_keepry   r9   )
r:   r  rk   r   r;   r  rv   r   r   r   )r<   r=   s   "€r3   r>   ÚPhi3ForCausalLM.__annotate__º  s»   ø€ ÷ 6
ñ 6
á×#Ñ# dÕ*ð6
ñ Ÿ™ tÕ+ð6
ñ ×&Ñ&¨Õ-ð	6
ñ
  �ð6
ñ ×(Ñ(¨4Õ/ð6
ñ × Ñ  4Õ'ð6
ñ ˜$•;ð6
ñ ™eŸl™lÕ*ð6
ñ Ñ+Õ,ð6
ñ 
 ñ6
r5   c	                ó–  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe-   V P                  ! RR	VR
VRV P                  P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      # )ai  
Example:

```python
>>> from transformers import AutoTokenizer, Phi3ForCausalLM

>>> model = Phi3ForCausalLM.from_pretrained("meta-phi3/Phi3-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-phi3/Phi3-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r>  r§   r�   rÕ   r?  r  Nr‚   rM  rR  r1  )ÚlossrM  rÕ   r8   r  r  )r  rA  r…   rv   ÚslicerK  Úloss_functionr(   r1  r   rÕ   r8   r  )r1   r>  r§   r�   rÕ   r?  rR  r  rS  ry   Úoutputsr8   Úslice_indicesrM  rV  s   &&&&&&&&&,     r3   rG   ÚPhi3ForCausalLM.forward¸  sÿ   € ð> ,0¯:ª:ñ ,
Øð,
à)ð,
ð &ð,
ð ,ð	,
ð
 (ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñp¨VÐp¸FÐpÈtÏ{É{×OeÑOeÐpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r5   c                óJ  <€ V'       d|   \        V P                  R 4      '       d`   VP                  ^,          V P                  P                  ^,           8¼  d-   VP	                  4       p	W�P                  P                  8:  d   Rp\
        SV `  ! R	RVRVRVRVRVRVRV/VB p
V
# )
Ú original_max_position_embeddingsNr>  rÕ   r§   r?  r�   r  rS  r  )Úhasattrr(   r„   r]  rC  r&   Úprepare_inputs_for_generation)r1   r>  rÕ   r§   r?  r�   r  rS  ry   Úpast_lengthÚmodel_inputsr2   s   &&&&&&&&,  €r3   r_  Ú-Phi3ForCausalLM.prepare_inputs_for_generationò  sÁ   ø€ ÷" Ü˜Ÿ™Ð%G×HÒHØ—‘ Õ" d§k¡k×&RÑ&RÐUVÕ&VÔVà)×8Ñ8Ó:ˆKØŸk™k×JÑJÔJØ"&�ä‘wÒ<ñ 	
Øð	
à+ð	
ð *ð	
ð (ð		
ð
 &ð	
ð  ð	
ð *ð	
ð ñ	
ˆð Ðr5   )rK  r  r1  )NNNNNNNr   )NNNNTr   )rJ   rK   rL   rM   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr'   r   r   rG   r_  rN   rO   rP   rQ   s   @@r3   rJ  rJ  ¨  sl   ù‡ € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Ø÷6
ñ 6
ó ó ð6
÷p#÷ #r5   rJ  c                   ó   € ] tR tRtRtR# )ÚPhi3ForSequenceClassificationi  r  N©rJ   rK   rL   rM   rN   r  r5   r3   rh  rh    ó   † Ûr5   rh  c                   ó   € ] tR tRtRtR# )ÚPhi3ForTokenClassificationi  r  Nri  r  r5   r3   rl  rl    rj  r5   rl  )r×   )é   )r  r*  rJ  rh  rl  )>Úcollections.abcr   r:   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   Úmasking_utilsr
   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_phi3r   rª   r!   rS   r™   r¡   r¸   rÅ   rÇ   rç   rþ   r  r*  rJ  rh  rl  Ú__all__r  r5   r3   Ú<module>r€     sM  ðõ, %ã Ý å !ß .Ý )Ý 7ß RÝ B÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ô)ˆb�i‰iô )ô$8<˜"Ÿ)™)ô 8<òv(õ	U÷%ô2ô<?)�B—I‘Iô ?)ñD ˜YÓ'ôJ�"—)‘)ó Jó (ðJô((Ð1ô (ðV ô˜/ó ó ðð& ôF
Ð#ó F
ó ðF
ðR ôlÐ)¨?ó ló ðlô^	Ð$DÐFYô 	ô	Ð!>Ð@Sô 	ò‚r5   