+
    œ°³j\(  ã                   óÐ  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIHt ^RI	H
t
 ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHtHtHtHtHt ^RIHt ^RIHt ]P>                  ! ] 4      t!Rt"Rt# ! R R]PH                  4      t% ! R R]4      t&R!R lt' ! R R]PH                  4      t( ! R R]4      t) ! R R]4      t* ! R R]4      t+ ! R R]4      t, ! R R ]4      t-. R"Ot.R# )#zPyTorch Phi-3 model.)ÚCallableN)Únn)ÚACT2FN)ÚCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úlogging)ÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralPreTrainedModelÚeager_attention_forwardÚrotate_half)ÚPhiRotaryEmbedding)Ú
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   óD   a a€ ] tR t^0t oV 3R ltV3R lR ltRtVtV ;t# )ÚPhi3MLPc                ó>  <€ \         SV `  4        Wn        \        P                  ! VP
                  ^VP                  ,          RR7      V n        \        P                  ! VP                  VP
                  RR7      V n        \        VP                  ,          V n        R# )é   F©ÚbiasN)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn)Úselfr   Ú	__class__s   &&€Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/phi3/modular_phi3.pyr   ÚPhi3MLP.__init__1   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÕ:VÐ]bÔcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Õ6ˆÖó    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úhidden_statesÚreturn)ÚtorchÚFloatTensor)ÚformatÚ__classdict__s   "€r&   Ú__annotate__ÚPhi3MLP.__annotate__9   s'   ø€ ÷ )ñ )¡U×%6Ñ%6ð )¹5×;LÑ;Lñ )r(   c                óž   € V P                  V4      pVP                  ^RR7      w  r2W P                  V4      ,          pV P                  V4      # )r   ©Údiméÿÿÿÿ)r    Úchunkr#   r!   )r$   r*   Ú	up_statesÚgates   &&  r&   ÚforwardÚPhi3MLP.forward9   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆØ× 2Ñ 2°4Ó 8Õ8ˆ	à�~‰~˜iÓ(Ð(r(   )r#   r   r!   r    ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r9   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r%   r/   s   @@r&   r   r   0   s   ù‡ € õ7÷)÷ )ð )r(   r   c                   ó   € ] tR t^BtRtR# )ÚPhi3RotaryEmbedding© N©r<   r=   r>   r?   r@   rF   r(   r&   rE   rE   B   ó   † Ûr(   rE   c                ó²  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜\        P                  ! Wb,          \	        V4      V,          ,           V.RR7      p
\        P                  ! W‚,          \	        V4      V,          ,           V	.RR7      pW«3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr3   r5   )Ú	unsqueezeÚshaper,   Úcatr   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds   &&&&&       r&   Úapply_rotary_pos_embrY   F   sÆ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cà—‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ä�iŠi˜%�+¬+°eÓ*<¸sÕ*BÕCÀVÐLÐRTÔU€GÜ�iŠi˜%�+¬+°eÓ*<¸sÕ*BÕCÀVÐLÐRTÔU€GØÐÐr(   c                   ó\   a a€ ] tR t^dt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )ÚPhi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# )r   r   Ú	layer_idxN©r   Úint)r.   r/   s   "€r&   r0   ÚPhi3Attention.__annotate__g   s#   ø€ ÷ Kñ K™zð K±c¸Dµjñ Kr(   c                ó¾  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        VP                  V n        V P                  R,          V n
        VP                  V n        RV n        VP                  V P                  ,          ^VP                  V P                  ,          ,          ,           p\        P                  ! VP                  V P                  ,          VP
                  RR7      V n        \        P                  ! VP
                  VRR7      V n        R# )Úhead_dimTFr   Ng      à¿)r   r   r   r]   Úgetattrr   Únum_attention_headsrb   Únum_key_value_headsÚnum_key_value_groupsÚscalingÚattention_dropoutÚ	is_causalr   r   Úo_projÚqkv_proj)r$   r   r]   Úop_sizer%   s   &&& €r&   r   ÚPhi3Attention.__init__g   sù   ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒà×,Ñ,¨t¯}©}Õ<¸qÀF×D^ÑD^Ðae×anÑanÕDnÕ?oÕoˆÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐejÔkˆŒÜŸ	š	 &×"4Ñ"4°gÀEÔJˆŽr(   c                óR  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r   r*   Úposition_embeddingsÚattention_maskNÚpast_key_valuesÚkwargsr+   )r,   ÚTensorÚtupler   r	   r   )r.   r/   s   "€r&   r0   r`   v   s—   ø€ ÷ -)ñ -)á—|‘|ð-)ñ #¡5§<¡<±·±Ð#=Õ>ð-)ñ Ÿ™ tÕ+ð	-)ñ
  �ð-)ñ Ñ-Õ.ð-)ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ-)r(   c                ó*  € VP                   R R p. VORNV P                  N5pV P                  V4      pV P                  P                  V P                  ,          p	VRR V	13,          p
VRW™V P
                  V P                  ,          ,           13,          pVRW�P
                  V P                  ,          ,           R 13,          pV
P                  V4      P                  ^^4      p
VP                  V4      P                  ^^4      pVP                  V4      P                  ^^4      pVw  rÞ\        W«WÞ4      w  r«Ve   VP                  W¼V P                  4      w  r¼\        P                  ! V P                  P                  \        4      pT! V V
VVV3RV P                  '       g   RMV P                   RV P"                  R\%        V P                  RR 4      /VB w  ppVP&                  ! . VORN5!  P)                  4       pV P+                  V4      pVV3# )N.Údropoutg        rg   Úsliding_windowr5   )rK   rb   rk   r   rd   re   ÚviewÚ	transposerY   Úupdater]   r   Úget_interfaceÚ_attn_implementationr   Útrainingrh   rg   rc   ÚreshapeÚ
contiguousrj   )r$   r*   ro   rp   rq   rr   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesÚ
key_statesÚvalue_statesrO   rP   Úattention_interfaceÚattn_outputÚattn_weightss   &&&&&,            r&   r9   ÚPhi3Attention.forwardv   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà�m‰m˜MÓ*ˆØ—K‘K×3Ñ3°d·m±mÕCˆ	Ø˜3 
  
˜?Õ+ˆØ˜˜i°d×6NÑ6NÐQU×Q^ÑQ^Õ6^Õ*^Ð^Ð^Õ_ˆ
Ø˜3 	×,DÑ,DÀtÇ}Á}Õ,TÕ TÑ VÐVÕWˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ô # 4§;¡;Ð0@À$ÓGð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r(   )
rh   r   rb   ri   r]   rf   re   rj   rk   rg   )N)
r<   r=   r>   r?   Ú__doc__r   r9   r@   rA   rB   rC   s   @@r&   r[   r[   d   s#   ù‡ € ÙG÷Kõ K÷-)÷ -)ò -)r(   r[   c                   óT   a a€ ] tR t^¦t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPhi3DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r   r]   r^   )r.   r/   s   "€r&   r0   ÚPhi3DecoderLayer.__annotate__§   s   ø€ ÷ @ñ @™zð @±cñ @r(   c                ó  <€ \         SV `  W4       Wn        \        WR 7      V n        \        V4      V n        \        P                  ! VP                  4      V n
        \        P                  ! VP                  4      V n        R# ))r   r]   N)r   r   r   r[   Ú	self_attnr   Úmlpr   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout)r$   r   r]   r%   s   &&&€r&   r   ÚPhi3DecoderLayer.__init__§   sZ   ø€ Ü‰Ñ˜Ô+ØŒÜ&¨fÔJˆŒÜ˜6“?ˆŒÜ"$§*¢*¨V×-?Ñ-?Ó"@ˆÔÜ!#§¢¨F×,>Ñ,>Ó!?ˆÖr(   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r   r*   rp   NÚposition_idsrq   Ú	use_cachero   rr   r+   )r,   rs   Ú
LongTensorr   Úboolrt   r	   r   )r.   r/   s   "€r&   r0   r�   ¯   sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ-Õ.ðñ 
�‰ñr(   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW€P                  V4      ,           pTpV P                  V4      pV P	                  V4      pW€P                  V4      ,           pV# )r*   rp   r™   rq   rš   ro   rF   )Úinput_layernormr‘   r•   Úpost_attention_layernormr’   r–   )
r$   r*   rp   r™   rq   rš   ro   rr   ÚresidualÚself_attn_weightss
   &&&&&&&,  r&   r9   ÚPhi3DecoderLayer.forward¯   s½   € ð !ˆØ×,Ñ,¨]Ó;ˆà+/¯>ª>ñ ,
Ø'ð,
à)ð,
ð &ð,
ð ,ð	,
ð
  ð,
ð !4ð,
ð ñ,
Ñ(ˆð !×#:Ñ#:¸=Ó#IÕIˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ ×#9Ñ#9¸-Ó#HÕHˆØÐr(   )r   r’   r•   r–   r‘   )NNNFNr;   rC   s   @@r&   r�   r�   ¦   s    ù‡ € ÷@ó @÷÷ ò r(   r�   c                   ó   € ] tR t^ÎtRtRtR# )ÚPhi3PreTrainedModelz0.0.5rF   N)r<   r=   r>   r?   Ú_versionr@   rF   r(   r&   r¤   r¤   Î   s   † Ø„Hr(   r¤   c                   ó*   a € ] tR t^Òt o RR ltRtV tR# )ÚPhi3ForCausalLMNc                óV  € V'       d|   \        V P                  R 4      '       d`   VP                  ^,          V P                  P                  ^,           8¼  d-   VP	                  4       p	W�P                  P                  8:  d   Rp\
        P                  ! V 3RVRVRVRVRVRVRV/VB p
V
# )	Ú original_max_position_embeddingsNÚ	input_idsrq   rp   Úinputs_embedsr™   rš   Úlogits_to_keep)Úhasattrr   rK   r©   Úget_seq_lengthr   Úprepare_inputs_for_generation)r$   rª   rq   rp   r«   r™   rš   r¬   rr   Úpast_lengthÚmodel_inputss   &&&&&&&&,  r&   r¯   Ú-Phi3ForCausalLM.prepare_inputs_for_generationÓ   sÆ   € ÷" Ü˜Ÿ™Ð%G×HÒHØ—‘ Õ" d§k¡k×&RÑ&RÐUVÕ&VÔVà)×8Ñ8Ó:ˆKØŸk™k×JÑJÔJØ"&�ä&×DÒDØñ

àð

ð ,ð

ð *ð	

ð
 (ð

ð &ð

ð  ð

ð *ð

ð ñ

ˆð Ðr(   rF   )NNNNTé    )r<   r=   r>   r?   r¯   r@   rA   )r/   s   @r&   r§   r§   Ò   s   ø‡ € ÷$ò $r(   r§   c                   ó   € ] tR t^útRtR# )ÚPhi3ForSequenceClassificationrF   NrG   rF   r(   r&   rµ   rµ   ú   rH   r(   rµ   c                   ó   € ] tR t^þtRtR# )ÚPhi3ForTokenClassificationrF   NrG   rF   r(   r&   r·   r·   þ   rH   r(   r·   )é   )r¤   Ú	Phi3Modelr§   rµ   r·   )/r‹   Úcollections.abcr   r,   r   Úactivationsr   Úcache_utilsr   Ú
generationr   Úmodeling_flash_attention_utilsr   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   Úmistral.modeling_mistralr   r   r   r   r   r   r   Úphi.modeling_phir   Úconfiguration_phi3r   Ú
get_loggerr<   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚModuler   rE   rY   r[   r�   r¤   r§   rµ   r·   Ú__all__rF   r(   r&   Ú<module>rË      sÙ   ðñ å $ã Ý å !Ý  Ý )Ý BÝ 5Ý &Ý ÷÷ ñ õ 2Ý *ð 
×	Ò	˜HÓ	%€à8Ð Ø€ô)ˆb�i‰iô )ô$	Ð,ô 	ôô<?)�B—I‘Iô ?)ôD%Ð*ô %ôPÐ0ô ô%Ð(ô %ôP	Ð$Dô 	ô	Ð!>ô 	ò‚r(   