+
    œ°³j•á  ã                   ó¢  € ^ RI Ht ^ RIHt ^ RIt^ RIHt ^ RIHu Ht	 ^RI
Ht ^RIHt ^RIHtHt ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHtHt ^RIHtH t  ^RI!H"t"H#t# ^RI$H%t% ^RI&H't'H(t(H)t)H*t*H+t+ ^RI,H-t- ^RI.H/t/H0t0 ^RI1H2t2 ^RI3H4t4 ^RI5H6t6H7t7H8t8  ! R R]Pr                  4      t: ! R R]Pr                  4      t;R R lt<RJR R llt=R t>R R lt? ! R R ]Pr                  4      t@ ! R! R"]4      tA ! R# R$]Pr                  4      tB]( ! R% R&]#4      4       tC]( ! R' R(]C4      4       tD ! R) R*]Pr                  4      tE ! R+ R,]Pr                  4      tF ! R- R.]Pr                  4      tG ! R/ R0]Pr                  4      tH ! R1 R2]Pr                  4      tI ! R3 R4]Pr                  4      tJRKR5 ltK ! R6 R7]Pr                  4      tL ! R8 R9]4      tMR: R; ltN]( ! R< R=]C4      4       tO](! R>R?7      ] ! R@ RA]4      4       4       tP](! RBR?7       ! RC RD]C4      4       tQ](! RER?7      ] ! RF RG]4      4       4       tR]( ! RH RI]C]4      4       tS. RLOtTR# )Mé    )ÚCallable)Ú	dataclassN)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_checkÚ	torch_int)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs)Úget_vision_position_ids)ÚStep3p7ConfigÚStep3p7TextConfigÚStep3p7VisionConfigc                   óÎ   a a€ ] tR t^.t oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       R	 4       tR
 tRtVtV ;t# )ÚStep3p7VisionRotaryEmbeddingzí
Simple axial 2D rope with same freqs used for H and W grids. The freqs are
pre-computed using `head-dim//4` which is later used to concat H and W positions.
The final angles rotate over the whole head dim, no partial rotation involved.
Údeviceú5.18©Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfig©r"   )ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/step3p7/modeling_step3p7.pyÚ__annotate__Ú)Step3p7VisionRotaryEmbedding.__annotate__6   s   ø€ ÷ Oñ OÑ2ñ Oó    c                óÎ  <€ \         SV `  4        Wn        V P                  P                  R ,          V n        V P
                  pV P                  R8w  d/   \        V P                  P                   RV P                   24      hV! V P                  V4      w  q@n	        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistentN)ÚsuperÚ__init__r,   Úrope_parametersr5   Úcompute_axial_rope_parametersÚ
ValueErrorÚ	__class__Ú__name__Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr,   r%   Úrope_init_fnrC   r>   s   &&&  €r0   r:   Ú%Step3p7VisionRotaryEmbedding.__init__5   s¬   ø€ ä‰ÑÔØŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!CÑ!CˆØ�>‰>˜WÔ$Ü §¡× 7Ñ 7Ð8Ð8aÐbf×bpÑbpÐaqÐrÓsÐsÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr3   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r+   r,   Úreturn)r"   ÚtupleÚtorchÚTensorÚfloat)r.   r/   s   "€r0   r1   r2   E   s+   ø€ ÷ 5ñ 5Ù#ð5á	‰u�|‰|™UÐ"Õ	#ñ5r3   c           	     óB  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pV^,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNç      ð?©Údtype)r;   ÚgetattrÚhidden_sizeÚnum_attention_headsrL   ÚarangerN   Úto)r,   r%   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorrC   s   &&,     r0   r<   Ú:Step3p7VisionRotaryEmbedding.compute_axial_rope_parametersC   s‡   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆØ˜Q•hˆàÐØ˜$¤5§<¢<°°;ÀÌÏÉÔ#UÐXcÕ#cÕdÕeˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                óZ  € VR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W0P                  P                  4       ,          pVP                  4       V P                  ,          pVP                  4       V P                  ,          pRRR4       V P                  X4      pV P                  X4      pWg3#   + '       g   i     L5; i).NÚmpsÚcpuF©Údevice_typeÚenabled).N)rN   Ú
isinstancer%   ÚtypeÚstrr   rC   Úcosr@   ÚsinÚrecomposition_frequencies)rF   ÚxÚposition_idsÚposition_ids_expandedrd   Úfreqsri   rj   s   &&&     r0   ÚforwardÚ$Step3p7VisionRotaryEmbedding.forwardY   sÓ   € ð !-¨YÕ 7× =Ñ =Ó ?ÐÜ'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ)¯M©M×,?Ñ,?Ó,AÕAˆEØ—)‘)“+ × 6Ñ 6Õ6ˆCØ—)‘)“+ × 6Ñ 6Õ6ˆC÷ Dð
 ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØˆxˆ÷ D×Cús   Â
A#DÄD*	c                óV   € VP                  ^4      p\        P                  ! W"3RR7      # )zM
Recompose the frequencies into the final spatial layout used per each grid.
©r\   éÿÿÿÿ)ÚflattenrL   Úcat)rF   ÚfreqÚfreq_hws   && r0   rk   Ú6Step3p7VisionRotaryEmbedding.recomposition_frequenciesg   s%   € ð —,‘,˜q“/ˆÜ�yŠy˜'Ð+°Ô4Ð4r3   )r@   r,   rC   rE   r5   ©N)r?   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r:   Ústaticmethodr<   rL   Úno_gradrp   rk   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r>   r/   s   @@r0   r$   r$   .   sy   ù‡ € ññ �X vÔ.÷Oô Oó /ðOð Ù�X vÔ.÷5ñ 5ó /ó ð5ð( ‡]‚]ƒ_ñó ð÷5ò 5r3   r$   c                   óD   a a€ ] tR t^ot oV 3R ltV3R lR ltRtVtV ;t# )ÚStep3p7VisionMLPc                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # rz   )r9   r:   r,   r   Ú
hidden_actÚactivation_fnrA   ÚLinearrV   Úintermediate_sizeÚfc1Úfc2©rF   r,   r>   s   &&€r0   r:   ÚStep3p7VisionMLP.__init__p   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r+   Úhidden_statesrJ   ©rL   rM   )r.   r/   s   "€r0   r1   ÚStep3p7VisionMLP.__annotate__w   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rz   )rŒ   r‰   r�   )rF   r’   s   &&r0   rp   ÚStep3p7VisionMLP.forwardw   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr3   )r‰   r,   rŒ   r�   ©	r?   r{   r|   r}   r:   rp   r�   r‚   rƒ   r„   s   @@r0   r†   r†   o   s   ù‡ € õK÷÷ ð r3   r†   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r+   r’   Ún_reprJ   ©rL   rM   Úint)r.   s   "r0   r1   r1   ~   s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur3   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)ºNNNr�   Nr�   r�   )ÚshapeÚexpandÚreshape)r’   r™   ÚbatchÚnum_key_value_headsÚslenrQ   s   &&    r0   Ú	repeat_kvr¤   ~   sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr3   c                ó   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\
        \        ,          /# )
r+   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutrZ   )rA   ÚModulerL   rM   rN   r   r   )r.   s   "r0   r1   r1   Š   sw   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô Ô'Õ(ñ%r3   c                 ó"  € \        W P                  4      p\        W0P                  4      p	\        P                  ! WP	                  ^^4      4      V,          p
Ve	   W¤,           p
\
        P                  P                  V
R\        P                  R7      P                  VP                  4      p
\
        P                  P                  W¦V P                  R7      p
\        P                  ! W©4      pVP	                  ^^4      P                  4       pWº3# )r+   )r\   rT   )ÚpÚtrainingrt   )r¤   Únum_key_value_groupsrL   ÚmatmulÚ	transposerA   Ú
functionalÚsoftmaxÚfloat32rY   rT   r¬   r°   Ú
contiguous)r¦   r§   r¨   r©   rª   r«   r¬   rZ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&,    r0   Úeager_attention_forwardr¼   Š   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r3   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrs   rt   )rž   rL   rv   )rl   Úx1Úx2s   &  r0   Úrotate_halfrÀ   £   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r3   c                óî   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# )r+   ÚqÚkri   rj   rJ   ©rL   rM   rK   )r.   s   "r0   r1   r1   ª   sV   € ÷ ñ Ü‡|�|ðÜŸ™ðÜ+0¯<©<ðÜ>C¿l¹lðä
Œ5�<‰<œŸ™Ð%Õ&ñr3   c                 ó°  € V P                   pVP                   pV P                  4       VP                  4       rVP                  R4      P                  4       VP                  R4      P                  4       r2W,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pVP	                  V4      pVP	                  V4      pWg3# )r+   éþÿÿÿ)rT   rN   Ú	unsqueezerÀ   rY   )rÂ   rÃ   ri   rj   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds   &&&&    r0   Úapply_rotary_pos_emb_visionrÌ   ª   s£   € ð —7‘7€LØ—7‘7€LØ�7‰7‹9�a—g‘g“i€qØ�}‰}˜RÓ ×&Ñ&Ó(¨#¯-©-¸Ó*;×*AÑ*AÓ*CˆØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØ�j‰j˜Ó&€GØ�j‰j˜Ó&€GØÐÐr3   c                   óX   a a€ ] tR t^¸t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚStep3p7VisionAttentionzÃCLIP-style vision attention; the only difference from [`CLIPAttention`] is
that queries and keys are rotated by the tower's 3D RoPE before the
(interface-dispatched) scaled dot-product attention.c                ó    <€ V ^8„  d   QhRS[ /# r*   r-   )r.   r/   s   "€r0   r1   Ú#Step3p7VisionAttention.__annotate__½   s   ø€ ÷ &ñ &Ñ2ñ &r3   c                ó¦  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  R,          V n        VP                  V n
        RV n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        \        P                  ! V P                  V P                  4      V n        ^V n        R# )ç      à?FNç      à¿)r9   r:   r,   rV   Ú	embed_dimrW   Ú	num_headsrQ   ÚscaleÚattention_dropoutr¬   Ú	is_causalrA   rŠ   Úk_projÚv_projÚq_projÚout_projr±   rŽ   s   &&€r0   r:   ÚStep3p7VisionAttention.__init__½   s×   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Õ8ˆŒØ—]‘] DÕ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜŸ	š	 $§.¡.°$·.±.ÓAˆŒð %&ˆÖ!r3   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r+   r’   Úposition_embeddingsrª   NrZ   rJ   )rL   rM   rK   r   r   )r.   r/   s   "€r0   r1   rÐ   Ï   sy   ø€ ÷ !8ñ !8á—|‘|ð!8ñ #¡5§<¡<±·±Ð#=Õ>ð!8ñ Ÿ™ tÕ+ð	!8ñ
 Ñ+Õ,ð!8ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ!8r3   c                óÐ  € VP                   RR p. VORNV P                  N5pV P                  V4      P                  V4      pV P	                  V4      P                  V4      pV P                  V4      P                  V4      P                  ^^4      p	Vw  r«\        WxW«4      w  rxVP                  ^^4      VP                  ^^4      r‡\        P                  ! V P                  P                  \        4      pT! V VVV	V3RV P                  RV P                  '       g   RMV P                  /VB w  rÞVP                   ! . VORN5!  P#                  4       pV P%                  V4      V3# )z#Input shape: Batch x Time x ChannelNr«   r¬   ç        rt   )rž   rQ   rÛ   ÚviewrÙ   rÚ   r³   rÌ   r   Úget_interfacer,   Ú_attn_implementationr¼   rÖ   r°   r¬   r    r·   rÜ   )rF   r’   rß   rª   rZ   Úinput_shapeÚhidden_shapeÚqueriesÚkeysÚvaluesri   rj   Úattention_interfacer»   rº   s   &&&&,          r0   rp   ÚStep3p7VisionAttention.forwardÏ   s`  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—+‘+˜mÓ,×1Ñ1°,Ó?ˆØ�{‰{˜=Ó)×.Ñ.¨|Ó<ˆØ—‘˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆà&‰ˆÜ3°GÀ3ÓL‰ˆØ×)Ñ)¨!¨QÓ/°·±ÀÀ1Ó1E�ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð —J‘Jð	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ�}‰}˜[Ó)¨<Ð7Ð7r3   )r,   r¬   rÔ   rQ   rØ   rÙ   rÕ   r±   rÜ   rÛ   rÖ   rÚ   rz   ©
r?   r{   r|   r}   r~   r:   rp   r�   r‚   rƒ   r„   s   @@r0   rÎ   rÎ   ¸   s#   ù‡ € ñ<÷&ó &÷$!8÷ !8ò !8r3   rÎ   c                   óT   a a€ ] tR t^ót oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚStep3p7VisionEncoderLayerc                ó    <€ V ^8„  d   QhRS[ /# r*   r-   )r.   r/   s   "€r0   r1   Ú&Step3p7VisionEncoderLayer.__annotate__ô   s   ø€ ÷ eñ eÑ2ñ er3   c                ó\  <€ \         SV `  4        Wn        \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  \        P                   ! VP                  4      ,          4      V n        \        P                  ! VP                  \        P                   ! VP                  4      ,          4      V n        R# )©ÚepsN)r9   r:   r,   rÎ   Ú	self_attnr†   ÚmlprA   Ú	LayerNormrV   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterÚ	ParameterÚlayer_scale_init_valuerL   ÚonesÚlambda_1Úlambda_2rŽ   s   &&€r0   r:   Ú"Step3p7VisionEncoderLayer.__init__ô   s¿   ø€ Ü‰ÑÔØŒÜ/°Ó7ˆŒÜ# FÓ+ˆŒÜ "§¢¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜŸš V×%BÑ%BÄUÇZÂZÐPV×PbÑPbÓEcÕ%cÓdˆŒÜŸš V×%BÑ%BÄUÇZÂZÐPV×PbÑPbÓEcÕ%cÓdˆŽr3   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[ P                  /# )r+   r’   rª   Nrß   rJ   rÄ   )r.   r/   s   "€r0   r1   rð   þ   s[   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	ñ 
�‰ñr3   c                ó  € TpV P                  V4      pV P                  ! V3R VRV/VB w  rV P                  V,          pW,           pTpV P                  V4      pV P	                  V4      pV P
                  V,          pW,           pV# )rª   rß   )rø   rô   rý   rù   rõ   rþ   )rF   r’   rª   rß   rZ   ÚresidualÚ_s   &&&&,  r0   rp   Ú!Step3p7VisionEncoderLayer.forwardþ   s    € ð !ˆØ×-Ñ-¨mÓ<ˆØŸ>š>Øñ
à)ð
ð !4ð
ð ñ	
Ñˆð Ÿ™¨Õ5ˆØ%Õ0ˆà ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™¨Õ5ˆØ%Õ0ˆØÐr3   )r,   rý   rþ   rù   rø   rõ   rô   ©NNr—   r„   s   @@r0   rî   rî   ó   s    ù‡ € ÷eó e÷÷ ò r3   rî   c                   óf   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltR	V3R lR lltRtVtV ;t	# )
ÚStep3p7VisionEmbeddingsi  c                ó    <€ V ^8„  d   QhRS[ /# r*   r-   )r.   r/   s   "€r0   r1   Ú$Step3p7VisionEmbeddings.__annotate__  s   ø€ ÷ jñ jÑ2ñ jr3   c                óŽ  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! VP                  VP                  VP                  VP                  R R7      V n
        V P
                  V P                  ,          ^,          V n        V P                  V n        \        P                  ! V P                  V P                  4      V n        \        P                  ! \         P"                  ! V P                  4      P%                  R4      R R7      V n        R# )F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasr7   N©é   rt   )r9   r:   r,   rV   rÔ   Ú
image_sizeÚ
patch_sizerA   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingrB   rL   rX   rŸ   rm   rŽ   s   &&€r0   r:   Ú Step3p7VisionEmbeddings.__init__  sç   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒÜ!ŸyšyØ×+Ñ+Ø×+Ñ+Ø×)Ñ)Ø×$Ñ$Øô 
ˆÔð !ŸO™O¨t¯©Õ>À1ÕDˆÔØ!×-Ñ-ˆÔÜ"$§,¢,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔÜŸIšI¤e§l¢l°4×3EÑ3EÓ&F×&MÑ&MÈgÓ&VÐchÔiˆÖr3   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r+   Ú
embeddingsÚheightÚwidthrJ   rš   )r.   r/   s   "€r0   r1   r	  ,  s5   ø€ ÷ ñ ±5·<±<ð Éð ÑUXð Ñ]b×]iÑ]iñ r3   c                óº  € V P                   P                  P                  ^ ,          pW P                  ,          pW0P                  ,          p\	        VR,          4      p\
        P                  P                  4       '       g2   WW8X  d,   Wg8X  d&   V P                   P                  P                  ^ 4      # V P                   P                  P                  ^ 4      pVP                  R,          p	VP                  ^WwV	4      pVP                  ^ ^^^4      p\        P                  P                  VWV3RRR7      pVP                  ^ ^^^4      P                  ^RV	4      pV# )aè  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing and no class embeddings.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
rÒ   ÚbilinearF)ÚsizeÚmodeÚalign_cornersrt   )r  Úweightrž   r  r   rL   ÚjitÚ
is_tracingrÇ   r    ÚpermuterA   r´   Úinterpolaterâ   )
rF   r  r  r  r  Ú
new_heightÚ	new_widthÚsqrt_num_positionsÚpatch_pos_embedr\   s
   &&&&      r0   Úinterpolate_pos_encodingÚ0Step3p7VisionEmbeddings.interpolate_pos_encoding,  s/  € ð ×/Ñ/×6Ñ6×<Ñ<¸QÕ?ˆØŸ™Õ.ˆ
ØŸ_™_Õ,ˆ	Ü& }°cÕ'9Ó:ÐÜ�y‰y×#Ñ#×%Ò%¨*Ô*JÈyÔOnØ×*Ñ*×1Ñ1×;Ñ;¸AÓ>Ð>Ø×1Ñ1×8Ñ8×BÑBÀ1ÓEˆØ×Ñ˜rÕ"ˆØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆÜŸ-™-×3Ñ3ØØÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆØÐr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r+   Úpixel_valuesrJ   )rL   ÚFloatTensorrM   )r.   r/   s   "€r0   r1   r	  H  s(   ø€ ÷ 
ñ 
¡E×$5Ñ$5ð 
ÑZ_×ZfÑZfñ 
r3   c                óp  € VP                   w   r4pV P                  P                  P                  pV P                  VP	                  VR 7      4      pVP                  ^4      P                  ^^4      pV'       d   W€P                  W„V4      ,           pV# W€P                  V P                  4      ,           pV# )rS   )
rž   r  r%  rT   rY   ru   r³   r.  r  rm   )	rF   r1  r.  r  r  r  Útarget_dtypeÚpatch_embedsr  s	   &&&      r0   rp   ÚStep3p7VisionEmbeddings.forwardH  s¥   € Ø*×0Ñ0Ñˆˆ1�eØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆ
ç#Ø#×&CÑ&CÀJÐX]Ó&^Õ^ˆJð Ðð $×&=Ñ&=¸d×>OÑ>OÓ&PÕPˆJØÐr3   )	r,   rÔ   r  r  r  r  r  r  rm   ©F)
r?   r{   r|   r}   r:   r.  rp   r�   r‚   rƒ   r„   s   @@r0   r  r    s*   ù‡ € ÷jó j÷&ð ÷8
÷ 
ò 
r3   r  c                   ó�   a a€ ] tR tRt oRtRtRR.tR.tRtRt	Rt
RtRt]P                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚStep3p7PreTrainedModeliU  ÚmodelTrî   ÚStep3p7DecoderLayerÚpast_key_valuesFc                óê  <€ \         SV `  V4       \        V\        4      '       dQ   \        P
                  ! VP                  \        P                  ! VP                  4      P                  R
4      4       R	# \        V\        4      '       d   \        P                  P                  VP                  VP                  P                   4       \        P                  P                  VP"                  VP                  P                   4       R	# \        V\$        4      '       d´   VP&                   F¡  pVP(                  pVP*                  V,          R8w  d   \,        VP*                  V,          ,          pV! VP                  VR7      w  rE\        P
                  ! \/        W R24      V4       \        P
                  ! \/        W R24      V4       K£  	  R	# \        V\0        4      '       d`   \/        V P                  RR4      p\        P2                  ! VP4                  RVR7       \        P2                  ! VP6                  RVR7       R	# \        V\8        4      '       d]   \/        V P                  RR4      p\        P2                  ! VP:                  RVR7       \        P<                  ! VP>                  4       R	# \        V\@        4      '       d#   \        P<                  ! VP:                  4       R	# R	# )r  Údefault©Ú
layer_typeÚ	_inv_freqÚ_original_inv_freqÚinitializer_rangeg{®Gáz”?rá   )ÚmeanÚstdNr  )!r9   Ú_init_weightsrf   r  ÚinitÚcopy_rm   rL   rX   r  rŸ   rî   rA   Ú	constant_rý   r,   rû   rþ   ÚStep3p7RotaryEmbeddingÚlayer_typesÚcompute_default_rope_parametersr5   r   rU   ÚStep3p7ExpertsÚnormal_Úgate_up_projÚ	down_projÚStep3p7TopKRouterr%  Úzeros_Úe_score_correction_biasÚStep3p7RMSNorm)rF   r¦   r@  rG   Úcurr_inv_freqr  rE  r>   s   &&     €r0   rF  Ú$Step3p7PreTrainedModel._init_weightsb  sñ  ø€ ä‰Ñ˜fÔ%Ü�fÔ5×6Ò6Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9MÑ9MÓ,N×,UÑ,UÐV]Ó,^Ö_Ü˜Ô 9×:Ò:Ü�G‰G×Ñ˜fŸo™o¨v¯}©}×/SÑ/SÔTÜ�G‰G×Ñ˜fŸo™o¨v¯}©}×/SÑ/SÖTÜ˜Ô 6×7Ò7Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÕ/°9Ô<Ü#6°v×7GÑ7GÈ
Õ7SÕ#T�LÙ#/°·±È*Ô#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ó 1ô ˜¤×/Ò/ô ˜$Ÿ+™+Ð':¸DÓAˆCÜ�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 1×2Ò2Ü˜$Ÿ+™+Ð':¸DÓAˆCÜ�LŠL˜Ÿ™¨S°cÕ:Ü�KŠK˜×6Ñ6Ö7Ü˜¤×/Ò/Ü�KŠK˜Ÿ™Ö&ñ 0r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r*   ©r    )r.   r/   s   "€r0   r1   Ú#Step3p7PreTrainedModel.__annotate__U  s   ø‡ ‚ áÑò r3   © )r?   r{   r|   r}   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_static_cacheÚ_supports_attention_backendrL   r€   rF  Ú__annotate_func__r�   r‚   rƒ   r„   s   @@r0   r9  r9  U  sd   ù‡ € ð  ÐØ&*Ð#Ø4Ð6KÐLÐØ#4Ð"5ÐØ ÐØ€NØÐØ!ÐØ"&Ðà
‡]‚]ƒ_ô'ó ð'÷ … r3   r9  c                   óŽ   a a€ ] tR tRt oRtR]R]/tV3R lV 3R llt]	! RR7      ]
V3R	 lR
 l4       4       tV3R ltRtVtV ;t# )ÚStep3p7VisionModeli�  u!  Vision encoder: patch embeddings â†’ 2-D RoPE transformer layers â†’ conv downsampler.

The rotary embedding (``self.rotary_emb``) and layer stack (``self.layers``) are
held directly on this module, following the Gemma4 convention of not wrapping
them in a separate ``Encoder`` submodule.
r’   Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# r*   r-   )r.   r/   s   "€r0   r1   ÚStep3p7VisionModel.__annotate__�  s   ø€ ÷ 
ñ 
Ñ2ñ 
r3   c                óz  <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R 7      V n        \        V4      V n
        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        \        P                   ! VP                  VP                  ^,          ^^^R7      V n        \        P                   ! VP                  ^,          VP                  ^,          ^^^R7      V n        V P'                  4        R# u upi )rò   )r  r  ÚpaddingN)r9   r:   r  r  rA   rö   rV   r÷   Úpre_layernormr$   Ú
rotary_embÚ
ModuleListÚrangeÚnum_hidden_layersrî   Úlayersr  Údownsampler1Údownsampler2Ú	post_init)rF   r,   r  r>   s   && €r0   r:   ÚStep3p7VisionModel.__init__�  sî   ø€ Ü‰Ñ˜Ô Ü1°&Ó9ˆŒÜŸ\š\¨&×*<Ñ*<À&×BWÑBWÔXˆÔÜ6°vÓ>ˆŒÜ—m’mÔPUÐV\×VnÑVnÔPoÓ$pÑPoÈ1Ô%>¸vÖ%FÑPoÑ$pÓqˆŒÜŸIšI f×&8Ñ&8¸&×:LÑ:LÈqÕ:PÐ^_ÐhiÐstÔuˆÔÜŸIšIØ×Ñ Õ" F×$6Ñ$6¸Õ$:ÈÐRSÐ]^ô
ˆÔð 	�‰Öùò %qs   Â	D8F)Útie_last_hidden_statesc                óP   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[/# )r+   r1  rZ   rJ   )rL   rM   r   r   r   )r.   r/   s   "€r0   r1   ri  ž  s2   ø€ ÷ [ñ [¡E§L¡Lð [¹FÑCUÕ<Vð [Ñ[jñ [r3   c                ó¢  € VP                   w  r4rVWPP                  P                  ,          pW`P                  P                  ,          pV P                  VR R7      p	V P                  V	4      p	\        P
                  ! ^Wx..V	P                  R7      p
\        V
^R7      pV P                  W›4      pV P                   F  pV! V	3RV/VB p	K  	  V	P                   R,          pV	P                  ^ ^^4      P                  W>Wx4      p	V P                  V	4      p	V P                  V	4      p	\        V	P                  ^4      P                  ^ ^^4      R7      # )T)r.  ©r%   )Úspatial_merge_sizerß   )Úlast_hidden_statert   )rž   r  r  rl  rL   Útensorr%   r   rm  rq  r(  râ   rr  rs  r   ru   )rF   r1  rZ   Ú
batch_sizer  r  r  Úgrid_hÚgrid_wÚhidden_stateÚgrid_thwrm   rß   ÚlayerÚchannelss   &&,            r0   rp   ÚStep3p7VisionModel.forwardœ  s5  € ð (4×'9Ñ'9Ñ$ˆ
�vØŸ?™?×5Ñ5Õ5ˆØŸ/™/×4Ñ4Õ4ˆØ—‘ |Èd�ÓSˆØ×)Ñ)¨,Ó7ˆô —<’< ! VÐ!4Ð 5¸l×>QÑ>QÔRˆÜ.¨xÈAÔNˆØ"Ÿo™o¨lÓIÐØ—[”[ˆEÙ  ÑaÐCVÐaÐZ`ÑaŠLñ !ð  ×%Ñ% bÕ)ˆØ#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÐPVÓ_ˆØ×(Ñ(¨Ó6ˆØ×(Ñ(¨Ó6ˆÜ°×1EÑ1EÀaÓ1H×1PÑ1PÐQRÐTUÐWXÓ1YÔZÐZr3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r*   r-   )r.   r/   s   "€r0   r1   ri  �  s   ø‡ ‚ ñ  Ñò r3   )rr  rs  r  rq  rl  rm  )r?   r{   r|   r}   r~   rî   rÎ   Ú_can_record_outputsr:   r   r   rp   rd  r�   r‚   rƒ   r„   s   @@r0   rf  rf  �  sT   ù‡ € ñð 	Ð2ØÐ,ðÐ÷

ó 
ñ ¨EÔ2Ø÷[ó ó 3ð[÷; … r3   rf  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )rJ  i´  r%   r&   r'   c                ó    <€ V ^8„  d   QhRS[ /# r*   rX  )r.   r/   s   "€r0   r1   Ú#Step3p7RotaryEmbedding.__annotate__¶  s   ø€ ÷ Uñ U™}ñ Ur3   c           
     óÞ  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        \        \        VP                  4      4      V n        / V n	        V P                   Fû  pV P
                  P                  V,          pVf   K&  VR,          V P                  V&   V P                  pV P                  V,          R8w  d   \        V P                  V,          ,          pV! V P
                  W#R7      w  rg\        W R2\        P                  ! VRR7      4       \        W R2\        P                  ! VP!                  4       RR7      4       \        W R2V4       Ký  	  R # )	Nr5   r>  r?  rA  Fr7   rB  Ú_attention_scaling)r9   r:   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr,   ÚsortedÚsetrK  r5   r;   rL  r   ÚsetattrrA   rB   rD   )	rF   r,   r%   r@  Úrope_paramsrG   rU  Úcurr_attention_scalingr>   s	   &&&     €r0   r:   ÚStep3p7RotaryEmbedding.__init__µ  s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÕAˆKØÒ"Ùà)4°[Õ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÕ)¨YÔ6Ü2°4·>±>À*Õ3MÕN�Ù4@ÀÇÁÈfÔ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ô4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÔ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTó +r3   c                ód   <€ V ^8„  d   QhRS[ RS[R,          RS[S[P                  S[3,          /# )r+   r,   r@  NrJ   )r    rh   rK   rL   rM   rN   )r.   r/   s   "€r0   r1   r‰  Í  s7   ø€ ÷ 5ñ 5Ùð5Ù8;¸d½
ð5á	‰u�|‰|™UÐ"Õ	#ñ5r3   c           	     ó¦  € V P                   V,          R,          pV P                   V,          P                  RR4      p\        V RR4      ;'       g    V P                  V P                  ,          p\        We,          4      pRpRV\        P                  ! ^ V^\        P                  R7      V,          ,          ,          p	V	P                  V4      V3# )a9  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`, *optional*):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
rP   Úpartial_rotary_factorrR   rQ   NrS   )
r;   ÚgetrU   rV   rW   r›   rL   rX   rN   rY   )
r,   r%   r@  rZ   r[   r—  rQ   r\   r^   rC   s
   &&&,      r0   rL  Ú6Step3p7RotaryEmbedding.compute_default_rope_parametersË  s²   € ð" ×%Ñ% jÕ1°,Õ?ˆà &× 6Ñ 6°zÕ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4×hÐh¸×8JÑ8JÈf×NhÑNhÕ8hˆÜ�(Õ2Ó3ˆàÐð ˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                óZ  € \        W R 24      p\        W R24      pVR
,          P                  VP                  ^ ,          R^4      P                  \        P
                  VP                  R7      pVR,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        Wg,          P                  ^^4      p	\        P                  ! W™3RR7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP                  VP                  R	7      XP                  VP                  R	7      3#   + '       g   i     LH; i)rA  r‹  N©rT   r%   ra   rb   Frc   rs   rS   )Nr�   Nrt   )r�   Nr�   )rU   rŸ   rž   rY   rL   rN   r%   rf   rg   rh   r   r³   rv   ri   rj   rT   )rF   rl   rm   r@  rC   r@   Úinv_freq_expandedrn   rd   ro   Úembri   rj   s   &&&&         r0   rp   ÚStep3p7RotaryEmbedding.forwardè  sJ  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �]Õ#×*Ñ*¨<×+=Ñ+=¸aÕ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÓoð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã:A FÆF*	)r,   rK  r�  rŽ  r5   rz   r  )r?   r{   r|   r}   r   r:   r   rL  rL   r€   r   rp   r�   r‚   rƒ   r„   s   @@r0   rJ  rJ  ´  su   ù‡ € Ù�X vÔ.÷Uô Uó /ðUð* Ù�X vÔ.÷5ñ 5ó /ó ð5ð6 ‡]‚]ƒ_Øñ<ó ó ÷<ð <r3   rJ  c                   óX   a a€ ] tR tRt oRtR	V3R lV 3R llltR tR tR tRt	Vt
V ;t# )
rT  iþ  zCGemma-style RMSNorm: normalizes in fp32 and scales by `weight + 1`.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r+   r\   ró   )r›   rN   )r.   r/   s   "€r0   r1   ÚStep3p7RMSNorm.__annotate__  s   ø€ ÷ 5ñ 5™Cð 5¡eñ 5r3   c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! V4      4      V n        R # rz   )r9   r:   ró   rA   rú   rL   Úzerosr%  )rF   r\   ró   r>   s   &&&€r0   r:   ÚStep3p7RMSNorm.__init__  s,   ø€ Ü‰ÑÔØŒÜ—l’l¤5§;¢;¨sÓ#3Ó4ˆŽr3   c                óž   € V\         P                  ! VP                  ^4      P                  RRR7      V P                  ,           4      ,          # )r+   T)Úkeepdimrt   )rL   ÚrsqrtÚpowrD  ró   )rF   rl   s   &&r0   Ú_normÚStep3p7RMSNorm._norm  s4   € Ø”5—;’;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÕIÓJÕJÐJr3   c                ó²   € V P                  VP                  4       4      pVR V P                  P                  4       ,           ,          pVP                  V4      # )rR   )r©  rN   r%  Útype_as)rF   rl   Úoutputs   && r0   rp   ÚStep3p7RMSNorm.forward	  sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Õ4Õ5ˆØ�~‰~˜aÓ Ð r3   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)rK   r%  rž   ró   ©rF   s   &r0   Ú
extra_reprÚStep3p7RMSNorm.extra_repr  s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r3   )ró   r%  )g�íµ ÷Æ°>)r?   r{   r|   r}   r~   r:   r©  rp   r±  r�   r‚   rƒ   r„   s   @@r0   rT  rT  þ  s'   ù‡ € ÙM÷5õ 5ò
Kò!÷=ò =r3   rT  c                   óH   a a€ ] tR tRt oRV 3R lltV3R lR ltRtVtV ;t# )Ú
Step3p7MLPi  c                óº  <€ \         SV `  4        Wn        VP                  V n        V'       d   VP                  MVP
                  V n        \        P                  ! V P                  V P
                  VP                  R 7      V n	        \        P                  ! V P                  V P
                  VP                  R 7      V n
        \        P                  ! V P
                  V P                  VP                  R 7      V n        \        VP                  ,          V n        VP                  '       d   VP                  V,          M^ ;'       g    \!        R4      V n        R# )©r  ÚinfN)r9   r:   r,   rV   Úshare_expert_dimr‹   rA   rŠ   Úmlp_biasÚ	gate_projÚup_projrP  r   rˆ   Úact_fnÚswiglu_limits_sharedrN   Úlimit)rF   r,   Ú	layer_idxÚis_shared_expertr>   s   &&&&€r0   r:   ÚStep3p7MLP.__init__  sô   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔß<L ×!8Ò!8ÐRX×RjÑRjˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŒð AG×@[×@[Ð@[�f×1Ñ1°)Ö<Ðab×sÐsÔglÐmrÓgsˆŽ
r3   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r+   rJ   r“   )r.   r/   s   "€r0   r1   ÚStep3p7MLP.__annotate__"  s   ø€ ÷ )ñ )™EŸL™Lñ )r3   c                ó  € V P                  V P                  V4      4      P                  V P                  R 7      pV P	                  V4      P                  V P                  ) V P                  R7      pV P                  W#,          4      # )©Úmax©ÚminrÇ  )r¼  rº  Úclampr¾  r»  rP  )rF   rl   ÚgateÚups   &&  r0   rp   ÚStep3p7MLP.forward"  sc   € Ø�{‰{˜4Ÿ>™>¨!Ó,Ó-×3Ñ3¸¿
¹
Ð3ÓCˆØ�\‰\˜!‹_×"Ñ"¨¯
©
 {¸¿
¹
Ð"ÓCˆØ�~‰~˜d�iÓ(Ð(r3   )r¼  r,   rP  rº  rV   r‹   r¾  r»  r7  r—   r„   s   @@r0   r´  r´    s   ù‡ € ÷t÷)÷ )ð )r3   r´  c                   ób   a a€ ] tR tRt oRtRtR
V 3R lltV3R lR ltV3R lR ltR	t	Vt
V ;t# )rM  i(  z2Collection of expert weights stored as 3D tensors.Tc                óF  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P                  ^V P                  ,          V P
                  4      4      V n        \        P                  ! \        P                  ! V P                  V P
                  V P                  4      4      V n        \        VP                  ,          V n        Vf   \#        R4      V n        R# TV n        R# )r+   Nr·  )r9   r:   Únum_local_expertsÚnum_expertsrV   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimrA   rú   rL   ÚemptyrO  rP  r   rˆ   r¼  rN   r¾  )rF   r,   Úswiglu_limitr>   s   &&&€r0   r:   ÚStep3p7Experts.__init__-  sÃ   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÕG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Õ/ˆŒØ%1Ò%9”U˜5“\ˆŽ
¸|ˆŽ
r3   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r+   r’   Útop_k_indexÚtop_k_weightsrJ   r“   )r.   r/   s   "€r0   r1   ÚStep3p7Experts.__annotate__7  s?   ø€ ÷ ñ Ù"Ÿ\™\ðÙ8=¿¹ðÙUZ×UaÑUaðá	�‰ñr3   c                ó6  € \         P                  ! V4      p\         P                  ! 4       ;_uu_ 4        \        P                  ! W P
                  R 7      P                  ^^^ 4      p\         P                  ! VP                  RR7      ^ 4      P                  4       pRRR4       X FÜ  pV^ ,          pWpP
                  8X  d   K  \         P                  ! XV,          4      w  r‰V P                  \        P                  ! W,          V P                  V,          4      4      p
\        P                  ! W P                  V,          4      W9VR3,          ,          p
VP                  ^ WšP!                  VP"                  4      4       KÞ  	  V#   + '       g   i     Lô; i))Únum_classesrs   N)rt   rÆ   )rL   Ú
zeros_liker€   ÚFÚone_hotrÑ  r(  ÚgreaterÚsumÚnonzeroÚwhereÚ_apply_gateÚlinearrO  rP  Ú
index_add_rY   rT   )rF   r’   rÙ  rÚ  ÚfinalÚmaskÚhitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrents   &&&&       r0   rp   ÚStep3p7Experts.forward7  s&  € ô × Ò  Ó/ˆÜ�]Š]�_�_Ü—9’9˜[×6FÑ6FÔG×OÑOÐPQÐSTÐVWÓXˆDÜ—-’- §¡¨X Ó 6¸Ó:×BÑBÓDˆC÷ ó ˆJØ# A�ˆJØ×-Ñ-Ô-ÙÜ#(§;¢;¨t°JÕ/?Ó#@Ñ ˆIØ×&Ñ&¤q§x¢x°Õ0HÈ$×J[ÑJ[Ð\fÕJgÓ'hÓiˆGÜ—h’h˜w¯©°zÕ(BÓCÀmÐ_hÐjnÐTnÕFoÕoˆGØ×Ñ˜Q 	¯:©:°e·k±kÓ+BÖCñ ð ˆ÷ �_ús   ´A(FÆF	c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r+   Úgate_uprJ   r“   )r.   r/   s   "€r0   r1   rÛ  H  s#   ø€ ÷ ñ ¡5§<¡<ð ±E·L±Lñ r3   c                óâ   € VP                  ^RR7      w  r#V P                  V4      P                  V P                  R7      pVP                  V P                  ) V P                  R7      pW#,          # )r+   rs   rÆ  rÈ  rt   )Úchunkr¼  rÊ  r¾  )rF   rñ  rË  rÌ  s   &&  r0   rå  ÚStep3p7Experts._apply_gateH  sZ   € Ø—=‘= ¨�=Ó+‰ˆØ�{‰{˜4Ó ×&Ñ&¨4¯:©:Ð&Ó6ˆØ�X‰X˜4Ÿ:™:˜+¨4¯:©:ˆXÓ6ˆØ�yÐr3   )r¼  rP  rO  rÒ  rÔ  r¾  rÑ  rz   )r?   r{   r|   r}   r~   Ú#_fp8_experts_clamp_after_activationr:   rp   rå  r�   r‚   rƒ   r„   s   @@r0   rM  rM  (  s-   ù‡ € Ù<à*.Ð'÷L÷ð ÷"÷ ð r3   rM  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )rQ  iO  c                ó    <€ V ^8„  d   QhRS[ /# r*   ©r!   )r.   r/   s   "€r0   r1   ÚStep3p7TopKRouter.__annotate__P  s   ø€ ÷ Xñ XÑ0ñ Xr3   c                ó„  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      4      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R # rz   )r9   r:   Únum_experts_per_tokÚtop_krÐ  rÑ  rV   rÒ  rA   rú   rL   rÕ  r%  rB   r£  rS  rŽ   s   &&€r0   r:   ÚStep3p7TopKRouter.__init__P  s{   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒÜ')§y¢y´·²¸V×=UÑ=UÓ1VÓ'WˆÖ$r3   c                ó�   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  S[ P                  3,          /# r‘   rÄ   )r.   r/   s   "€r0   r1   rù  X  s:   ø€ ÷ 	9ñ 	9¡U§\¡\ð 	9±e¹E¿L¹LÉ%Ï,É,ÑX]×XdÑXdÐ<dÕ6eñ 	9r3   c                óÖ  € VP                  RV P                  4      p\        P                  ! VP	                  V P
                  P                  4      V P
                  4      p\        P                  ! VP                  4       4      pW0P                  ,           p\        P                  ! W@P                  RRR7      w  rVVP                  ^V4      pWwP                  RRR7      ,          pW'V3# )r  F)r\   r�  T)r\   r¦  rt   )r    rÒ  rß  ræ  rY   r%  rT   ÚsigmoidrN   rS  rL   Útopkrü  Úgatherrâ  )rF   r’   Úrouter_logitsÚrouting_weightsÚscores_for_choicer  rÙ  rÚ  s   &&      r0   rp   ÚStep3p7TopKRouter.forwardX  s¶   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ×!1Ñ!1°$·+±+×2CÑ2CÓ!DÀdÇkÁkÓRˆäŸ)š) M×$7Ñ$7Ó$9Ó:ˆØ+×.JÑ.JÕJÐÜŸšÐ$5·z±zÀrÐRWÔX‰ˆØ'×.Ñ.¨q°+Ó>ˆØ×*Ñ*¨r¸4Ð*Ó@Õ@ˆØ¨[Ð8Ð8r3   )rS  rÒ  rÑ  rü  r%  r—   r„   s   @@r0   rQ  rQ  O  s    ù‡ € ÷Xó X÷	9÷ 	9ð 	9r3   rQ  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚStep3p7SparseMoeBlockid  c                ó  <€ \         SV `  4        VP                  '       d   VP                  V,          ;'       g    R MR p\        V4      V n        \        WR7      V n        \        WRR7      V n        VP                  V n
        R # )N)rÖ  T)rÀ  )r9   r:   Úswiglu_limitsrQ  rË  rM  Úexpertsr´  Úshared_expertsÚmoe_router_scaling_factorÚrouted_scaling_factor)rF   r,   r¿  rÖ  r>   s   &&& €r0   r:   ÚStep3p7SparseMoeBlock.__init__e  sn   ø€ Ü‰ÑÔð EK×DX×DXÐDX˜×,Ñ,¨YÕ7×?Ð?¸4øÐ^bˆÜ% fÓ-ˆŒ	Ü% fÔHˆŒÜ(¨ÈTÔRˆÔØ%+×%EÑ%EˆÖ"r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r‘   r“   )r.   r/   s   "€r0   r1   Ú"Step3p7SparseMoeBlock.__annotate__o  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ó  € VP                   w  r#pVP                  RV4      pV P                  V4      pV P                  V4      w  rgpV P	                  WV4      pWP
                  ,          pW,           pVP                  W#V4      pV# r  )rž   râ   r  rË  r  r  r    )	rF   r’   r}  Úsequence_lengthrÒ  Úshared_outputr  r  Úselected_expertss	   &&       r0   rp   ÚStep3p7SparseMoeBlock.forwardo  s†   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆÐ,ØŸ™ ]ÀoÓVˆà%×(BÑ(BÕBˆØ%Õ5ˆà%×-Ñ-¨jÈ:ÓVˆØÐr3   )r  rË  r  r  r—   r„   s   @@r0   r  r  d  s   ù‡ € õF÷÷ ð r3   r  c                ó¶  € VP                  V4      pVP                  V4      pVP                  R,          pV RRV13,          V RVR13,          rvVRRV13,          VRVR13,          r˜Wb,          \        V4      V,          ,           p
W‚,          \        V4      V,          ,           p\        P                  ! W§.RR7      p
\        P                  ! W¹.RR7      pW«3# )a{  Applies Rotary Position Embedding to the query and key tensors.

Removes the interleaving of cos and sin from GLM

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nrs   rt   )rÇ   rž   rÀ   rL   rv   )rÂ   rÃ   ri   rj   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passrÊ   rË   s   &&&&&       r0   Úapply_rotary_pos_embr    sÐ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2•€JØ�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6Ø�c˜;˜J˜;Ð&Õ'¨¨3°
±Ð+;Õ)<ˆ6ð �{œ{¨5Ó1°CÕ7Õ8€GØ�{œ{¨5Ó1°CÕ7Õ8€Gô �iŠi˜Ð)¨rÔ2€GÜ�iŠi˜Ð)¨rÔ2€GØÐÐr3   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚStep3p7Attentioni¥  zTAfmoe-style SWA/GQA attention with Step3p7-specific gating and per-layer head count.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r+   r,   r¿  )r!   r›   )r.   r/   s   "€r0   r1   ÚStep3p7Attention.__annotate__¨  s    ø€ ÷ Lñ LÑ0ð L¹Sñ Lr3   c                óø  <€ \         SV `  4        VP                  pW0n        Wn        W n        \        VR VP                  VP                  ,          4      V n        V P                  VP                  ,          V n
        VP                  R,          V n        VP                  V n        RV n        \        P                   ! VP                  V P                  V P                  ,          VP"                  R7      V n        \        P                   ! VP                  VP                  V P                  ,          VP"                  R7      V n        \        P                   ! VP                  VP                  V P                  ,          VP"                  R7      V n        \        P                   ! V P                  V P                  ,          VP                  VP"                  R7      V n        VP,                  V,          R8H  V n        V P.                  '       d   VP0                  MRV n        \3        V P                  VP4                  R7      V n        \3        V P                  VP4                  R7      V n        VP:                  RJ ;'       g    VP:                  R8H  V n        V P<                  '       d   V P                  MV P                  V P                  ,          p\        P                   ! VP                  VRR7      V n        R# )	rQ   Tr¶  Úsliding_attentionNrò   zper-headFrÓ   ) r9   r:   rW   rÕ   r,   r¿  rU   rV   rQ   r¢   r±   Úquery_pre_attn_scalarr«   r×   rØ   rA   rŠ   Úattention_biasrÛ   rÙ   rÚ   Úo_projrK  Úis_local_attentionÚsliding_windowrT  Úrms_norm_epsÚq_normÚk_normÚgatingÚgate_per_headÚg_proj)rF   r,   r¿  rÕ   Ú
g_proj_dimr>   s   &&&  €r0   r:   ÚStep3p7Attention.__init__¨  s  ø€ Ü‰ÑÔØ×.Ñ.ˆ	à"ŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$(§N¡N°f×6PÑ6PÕ$PˆÔ!Ø×3Ñ3°TÕ9ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’i × 2Ñ 2°D·N±NÀTÇ]Á]Õ4RÐY_×YnÑYnÔoˆŒÜ—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’i §¡°·±Õ >À×@RÑ@RÐY_×YnÑYnÔoˆŒð #)×"4Ñ"4°YÕ"?ÐCVÑ"VˆÔØ7;×7N×7NÐ7N˜f×3Ò3ÐTXˆÔä$ T§]¡]¸×8KÑ8KÔLˆŒÜ$ T§]¡]¸×8KÑ8KÔLˆŒØ#Ÿ]™]¨dÐ2×QÐQ°f·m±mÀzÑ6QˆÔØ'+×'9×'9Ð'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Õ?]ˆ
Ü—i’i × 2Ñ 2°JÀUÔKˆŽr3   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r+   r’   rß   rª   Nr<  rZ   rJ   )rL   rM   rK   r   r   r   )r.   r/   s   "€r0   r1   r"  È  s‡   ø€ ÷ *)ñ *)á—|‘|ð*)ñ #¡5§<¡<±·±Ð#=Õ>ð*)ñ Ÿ™ tÕ+ð	*)ñ
  �ð*)ñ Ñ+Õ,ð*)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ*)r3   c           	     óL  € VP                   R R p. VORNV P                  N5pV P                  V P                  V4      P	                  V4      4      P                  ^^4      pV P                  V P                  V4      P	                  V4      4      P                  ^^4      p	V P                  V4      P	                  V4      P                  ^^4      p
V P                  V4      pVw  rÍ\        W‰WÍ4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                   \"        4      pT! V VV	V
V3RV P$                  '       g   RMV P&                  RV P(                  RV P*                  /VB w  ppVP,                  ! . VORN5!  pVP                   pVP                  ! . VR R OV P.                  NV P                  N5!  pWûP1                  R4      P3                  4       ,          pVP                  ! V!  pV P5                  V4      pVV3# )Nr¬   rá   r«   r)  rt   )rž   rQ   r+  rÛ   râ   r³   r,  rÙ   rÚ   r/  r  Úupdater¿  r   rã   r,   rä   r¼   r°   r×   r«   r)  r    rÕ   rÇ   r   r'  )rF   r’   rß   rª   r<  rZ   rå   ræ   Úquery_statesr¸   r¹   Úgate_statesri   rj   rê   r»   rº   Úoutput_shapes   &&&&&,            r0   rp   ÚStep3p7Attention.forwardÈ  s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—k‘k -Ó0ˆØ&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð
%
ð —L‘Lð
%
ð  ×.Ñ.ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;ˆØ"×(Ñ(ˆØ!×&Ò&ÐY¨°S°bÐ(9ÐY¸4¿>¹>ÐYÈ4Ï=É=ÓYˆØ!×$9Ñ$9¸"Ó$=×$EÑ$EÓ$GÕGˆØ!×&Ò&¨Ñ5ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r3   )r×   r,   r/  r.  rQ   rØ   r(  r,  rÙ   r¿  rÕ   r±   r'  r+  rÛ   r«   r)  rÚ   rz   rì   r„   s   @@r0   r   r   ¥  s$   ù‡ € Ù^÷Ló L÷@*)÷ *)ò *)r3   r   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )r;  iõ  c                óÖ  <€ \         SV `  4        VP                  V n        VP                  V,          p\	        W24      V n        WP
                  n        VP                  V,          V n        VP                  V,          R 8X  d   \        M\        pV! W4      V n        \        VP                  VP                  R7      V n        \        VP                  VP                  R7      V n        R# )Úsparserò   N)r9   r:   rV   Úper_layer_configr   rô   r,   rK  Úattention_typeÚmlp_layer_typesr  r´  rõ   rT  r*  Úinput_layernormÚpost_attention_layernorm)rF   r,   r¿  Úlayer_configÚ	mlp_classr>   s   &&&  €r0   r:   ÚStep3p7DecoderLayer.__init__ö  s·   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔð ×.Ñ.¨yÕ9ˆÜ)¨,ÓBˆŒØ &�‰ÔØ$×0Ñ0°Õ;ˆÔð .4×-CÑ-CÀIÕ-NÐRZÔ-ZÕ)Ô`jˆ	Ù˜VÓ/ˆŒä-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÖ%r3   c                ó.  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          R	S[ P                  /# )
r+   r’   rª   Nrm   r<  Ú	use_cacherß   rZ   rJ   )rL   rM   Ú
LongTensorr   ÚboolrK   r   r   )r.   r/   s   "€r0   r1   Ú Step3p7DecoderLayer.__annotate__	  sœ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ ˜$•;ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEðñ Ñ+Õ,ðñ 
�‰ñr3   c                óÔ   € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  rW�,           pTpV P                  V4      pV P                  V4      pW�,           pV# )r’   rª   rm   r<  rE  rß   rZ  )r?  rô   r@  rõ   )
rF   r’   rª   rm   r<  rE  rß   rZ   r  r  s
   &&&&&&&,  r0   rp   ÚStep3p7DecoderLayer.forward	  s©   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆØÐr3   )r=  rV   r?  rõ   r@  rô   )NNNFNr—   r„   s   @@r0   r;  r;  õ  s   ù‡ € õd÷&÷ ò r3   r;  c                ór   € V ^8„  d   QhR\         R\        \         \         \         \         .\        3,          /# )r+   r)  rJ   )r›   r   rG  )r.   s   "r0   r1   r1   )  s/   € ÷ 
ñ 
´#ð 
¼(ÄCÌÌcÔSVÐCWÔY]ÐC]Õ:^ñ 
r3   c                ó   a € R V 3R llpV# )z9
Enables a bidirectional mask within the sliding window.
c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# )r+   Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrJ   )r›   rG  )r.   s   "r0   r1   Ú3_bidirectional_window_overlay.<locals>.__annotate__.  s/   € ÷ 4ñ 4œcð 4¬Sð 4¼ð 4Äcð 4Ìdñ 4r3   c                ó,   <€ \        W#,
          4      S8  # )z‹A token can attend to any other token if their absolute distance is within
the (exclusive) sliding window size (distance < sliding_window).)Úabs)rN  rO  rP  rQ  r)  s   &&&&€r0   Ú
inner_maskÚ1_bidirectional_window_overlay.<locals>.inner_mask.  s   ø€ ô �5•>Ó" ^Ñ3Ð3r3   rZ  )r)  rU  s   f r0   Ú_bidirectional_window_overlayrW  )  s   ø€ ÷
4ð 4ð
 Ðr3   c                   óŽ   a a€ ] tR tRt oR
tR]R]/tV3R lV 3R llt]	]
]RV3R lR ll4       4       4       tV3R ltR	tVtV ;t# )ÚStep3p7TextModeli6  r’   rg  c                ó    <€ V ^8„  d   QhRS[ /# r*   rø  )r.   r/   s   "€r0   r1   ÚStep3p7TextModel.__annotate__?  s   ø€ ÷ ñ Ñ0ñ r3   c           	     ó$  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \#        VR7      V n        RV n        VP(                  '       dr   \        VP                  VP                  VP(                  ,           4      pV Uu0 uF	  pRV R2kK  	  pp\+        V P,                  ;'       g    . 4      V,          V n        V P/                  4        R# u upi u upi )rò   ©r,   Fz(^|\.)layers\.z\.N)r9   r:   Úpad_token_idÚpadding_idxÚ
vocab_sizerA   r  rV   Úembed_tokensrn  ro  rp  r;  rq  rT  r*  ÚnormrJ  rm  Úgradient_checkpointingÚnum_nextn_predict_layersr�  Ú"_keys_to_ignore_on_load_unexpectedrt  )rF   r,   ÚiÚ
mtp_layersÚpatternsr>   s   &&   €r0   r:   ÚStep3p7TextModel.__init__?  s?  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÌUÐSY×SkÑSkÔMlÓ$mÑMlÈÔ%8¸Ö%CÑMlÑ$mÓnˆŒÜ" 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð ×*×*Ð*ô ˜v×7Ñ7¸×9QÑ9QÐTZ×TsÑTsÕ9sÓtˆJÙ9CÓD¹°A˜>¨!¨¨BÓ/¹ˆHÐDÜ69¸$×:aÑ:a×:gÐ:gÐegÓ6hÐksÕ6sˆDÔ3ð 	�‰Öùò! %nùò Es   ÂFÄ:Fc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
r+   Ú	input_idsNrª   rm   r<  Úinputs_embedsrE  rZ   rJ   )	rL   rF  rM   r   r2  rG  r   r   r   )r.   r/   s   "€r0   r1   r[  Y  s�   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ Ÿ™ tÕ+ðC
ñ ×&Ñ&¨Õ-ð	C
ñ
  �ðC
ñ ×(Ñ(¨4Õ/ðC
ñ ˜$•;ðC
ñ Ñ+Õ,ðC
ñ 
!ñC
r3   c                ót  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       gƒ   RV P                  RVRVRVRV/p
V
P                  4       pV P                  P                  '       d)   R	 V
R
&   \        V P                  P                  4      VR
&   R\!        R/ V
B R\#        R/ VB /p	Tp/ p\%        V P                  P&                  4       F  pV P)                  WÃV4      WÞ&   K  	  \+        V P,                  R V P                  P.                   4       FY  w  ppV! V3RW�P                  P&                  V,          ,          RWÐP                  P&                  V,          ,          RVRV/VB pK[  	  V P1                  V4      p\3        VVR7      # )Nz:You must specify exactly one of input_ids or inputs_embedsr]  ry  r,   rl  rª   r<  rm   c                  óN   € \         P                  ! R \         P                  R7      # )TrS   )rL   r|  rG  )Úargss   *r0   Ú<lambda>Ú*Step3p7TextModel.forward.<locals>.<lambda>~  s   € ÄÇÂÈTÔY^×YcÑYcÕ@dr3   Úor_mask_functionÚfull_attentionr$  rß   )r{  r<  rZ  )r=   ra  r   r,   Úget_seq_lengthrL   rX   rž   r%   rÇ   rf   ÚdictÚcopyÚuse_bidirectional_attentionrW  r)  r
   r   r�  rK  rm  Ú	enumeraterq  rp  rb  r   )rF   rk  rª   rm   r<  rl  rE  rZ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsr’   rß   r@  rf  Údecoder_layers   &&&&&&&,         r0   rp   ÚStep3p7TextModel.forwardV  s>  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFð ˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6×6Ð6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%]ÐI\Ñ%]ð#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAˆ}Ù)Øñà2·;±;×3JÑ3JÈ1Õ3MÕNðð %8¿¹×8OÑ8OÐPQÕ8RÕ$Sðð *ð	ð
 !0ðð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r*   rø  )r.   r/   s   "€r0   r1   r[  6  s   ø‡ ‚ áÑò r3   )re  ra  rc  rq  rb  r_  rm  r`  )Útext)NNNNNN)r?   r{   r|   r}   Úinput_modalitiesr;  r   r†  r:   r   r   r   rp   rd  r�   r‚   rƒ   r„   s   @@r0   rY  rY  6  s_   ù‡ € ð !ÐàÐ,ØÐ&ðÐ÷
ó ð.  ØØ÷C
ñ C
ó ó ó  ðC
÷G … r3   rY  zJ
    Base class for Llava outputs, with hidden states and attentions.
    )Úcustom_introc                   ó4   a € ] tR tRt o RtRtV 3R ltRtV tR# )ÚStep3p7ModelOutputWithPastiŸ  a¯  
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
Nc                óH   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   # )r+   NÚimage_hidden_states)rL   r2  )r.   r/   s   "€r0   r1   Ú'Step3p7ModelOutputWithPast.__annotate__Ÿ  s   ø‡ ‚ ñ$ ×*Ñ*¨TÕ1Ñ8ò% r3   rZ  )	r?   r{   r|   r}   r~   r†  rd  r�   r‚   ©r/   s   @r0   r„  r„  Ÿ  s   ø‡ € ñ	ð 59Ð÷% ƒ r3   r„  zw
    The Llava-Next model which consists of a vision backbone and a language model without language modeling head.
    c                   ó´   a a€ ] tR tRt oRtV3R lV 3R llt]]RV3R lR ll4       4       tV3R lR lt	]]RV3R	 lR
 ll4       4       t
V3R ltRtVtV ;t# )ÚStep3p7Modeli´  r:  c                ó    <€ V ^8„  d   QhRS[ /# r*   rX  )r.   r/   s   "€r0   r1   ÚStep3p7Model.__annotate__½  s   ø€ ÷ ñ ™}ñ r3   c                ó´  <€ \         SV `  V4       \        VP                  4      V n        \        VP                  4      V n        VP                  P                  V n        \        P                  ! VP                  P                  ^,          VP                  P                  VP                  R7      V n        VP                  V n        V P!                  4        R# )é   r¶  N)r9   r:   rf  Úvision_configÚvision_modelrY  Útext_configÚlanguage_modelr`  rA   rŠ   rV   Úprojector_biasÚmulti_modal_projectorÚimage_token_idÚimage_placeholder_token_idrt  rŽ   s   &&€r0   r:   ÚStep3p7Model.__init__½  s¡   ø€ Ü‰Ñ˜Ô Ü.¨v×/CÑ/CÓDˆÔÜ.¨v×/AÑ/AÓBˆÔØ ×,Ñ,×7Ñ7ˆŒô &(§Y¢YØ× Ñ ×,Ñ,¨qÕ0°&×2DÑ2D×2PÑ2PÐW]×WlÑWlô&
ˆÔ"ð +1×*?Ñ*?ˆÔ'Ø�‰Ör3   c          	      ó–   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          R,          RS[S[,          /# )r+   r1  Úpixel_values_localNÚnum_local_patchesrJ   )rL   rM   Úlistr›   rK   r   )r.   r/   s   "€r0   r1   rŒ  Ì  sO   ø€ ÷ )
ñ )
á—l‘lð)
ñ "ŸL™L¨4Õ/ð)
ñ  ¡�9 tÕ+ð	)
ñ 
Ñ+Õ	+ñ)
r3   c           	     ó  € V P                   ! V3/ VB pV P                  VP                  4      pVe3   V P                  V P                   ! V3/ / VCRR/CB P                  4      MRpVf   ^ .VP                  ^ ,          ,          p. p^ p	\	        V4       FÈ  w  r«. pV^ 8”  d=   WyW›,            pVP                  VP                  RVP                  R,          4      4       VP                  Wj,          P                  RVP                  R,          4      4       W›,          p	TP                  \        V4      ^8”  d   \        P                  ! V4      MV^ ,          4       KÊ  	  \        VP                  VVP                  VP                  R7      # )a+  
pixel_values_local (`torch.FloatTensor` of shape `(total_patches, 3, height, width)`, *optional*):
    All local patches flattened across the batch, or `None` if no local views.
num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
    Number of local patches per image, e.g. `[6, 0, 4]`.
NÚoutput_hidden_statesF)r{  Úpooler_outputr’   rg  rt   )r�  r”  r{  rž   rx  Úappendrâ   ÚlenrL   rv   r   r’   rg  )rF   r1  r™  rš  rZ   Úvision_outputÚimage_featuresÚpatch_image_featuresÚmergedÚcur_patch_idxrf  Ú	num_patchÚcur_featureÚpatch_slices   &&&&,         r0   Úget_image_featuresÚStep3p7Model.get_image_featuresÊ  s{  € ð ×)Ò)¨,ÑA¸&ÑAˆØ×3Ñ3°M×4SÑ4SÓTˆð
 "Ò-ð ×&Ñ&Ø×!Ò!Ð"4ÑbÐ8a¸6Ð8aÐCYÐ[`Ñ8aÑb×tÑtôð ð 	ð Ò$Ø!"  n×&:Ñ&:¸1Õ&=Õ =ÐàˆØˆÜ%Ð&7Ö8‰LˆAØˆKØ˜1Œ}Ø2À=ÕC\Ð]�Ø×"Ñ" ;×#3Ñ#3°B¸×8IÑ8IÈ"Õ8MÓ#NÔOØ×Ñ˜~Õ0×5Ñ5°b¸.×:NÑ:NÈrÕ:RÓSÔTØÕ&ˆMØ�M‰M´C¸Ó4DÀqÔ4Hœ%Ÿ)š) KÔ0ÈkÐZ[ÍnÖ]ñ 9ô *Ø+×=Ñ=Ø Ø'×5Ñ5Ø$×/Ñ/ô	
ð 	
r3   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r+   rk  rl  r¢  )rL   rF  r2  )r.   r/   s   "€r0   r1   rŒ  ÷  s7   ø€ ÷ "ñ "Ù×)Ñ)ð"Ù:?×:KÑ:Kð"Ù]b×]nÑ]nñ"r3   c           
     ó$  € Vfo   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pMWP                  P                  8H  pVP                  4       pVP                  R4      P                  VP                  4      p\        WRP                  R,          ,          VP                  4       8H  RV RVP                  ^ ,           24       V# )z×
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
r›  z6Image features and image tokens do not match, tokens: z, features: rZ  rt   )Úget_input_embeddingsrL   Úfullr,   r•  Úlongr%   Úallrâ  rÇ   rY   r   rž   Únumel)rF   rk  rl  r¢  Úspecial_image_maskÚn_image_tokenss   &&&&  r0   Úget_placeholder_maskÚ!Step3p7Model.get_placeholder_mask÷  sï   € ð ÒØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ×0Ñ0°Õ4Õ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`Ðao×auÑauÐvwÕaxÐ`yÐzô	
ð "Ð!r3   c                ó¦  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          S[ P
                  ,          R,          RS[ P
                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[R,          RS[S[	,          RS[
S[,          /# )r+   rk  Nr1  r™  rš  rª   rm   r<  rl  rE  rZ   rJ   )rL   rF  r2  r›  r›   rM   r   rG  r   r   rK   r„  )r.   r/   s   "€r0   r1   rŒ    sß   ø€ ÷ 0
ñ 0
á×#Ñ# dÕ*ð0
ñ ×'Ñ'¨$Õ.ð0
ñ "×-Ñ-°Õ4ð	0
ñ
  ¡�9¡u§|¡|Õ3°dÕ:ð0
ñ Ÿ™ tÕ+ð0
ñ ×&Ñ&¨Õ-ð0
ñ  �ð0
ñ ×(Ñ(¨4Õ/ð0
ñ ˜$•;ð0
ñ Ñ+Õ,ð0
ñ 
Ñ+Õ	+ñ0
r3   c
                óè  € Vf   V P                  4       ! V4      pRpVe~   V P                  W#VRR7      P                  p\        P                  ! V^ R7      P                  VP                  VP                  4      pV P                  WV4      pVP                  WË4      pV P                  ! RRRRVRVRVR	VR
V	/V
B p\        VP                  VP                  VP                  VP                  VR7      # )úì
pixel_values_local (`torch.FloatTensor`, *optional*):
    Local patch pixel values of shape `(total_patches, 3, H, W)`.
num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
    Number of local patches per image in the batch.
NT)Úreturn_dictrs   rk  rl  rª   rm   r<  rE  )r{  r<  r’   rg  r†  rZ  )r­  r©  rž  rL   rv   rY   r%   rT   r´  Úmasked_scatterr’  r„  r{  r<  r’   rg  )rF   rk  r1  r™  rš  rª   rm   r<  rl  rE  rZ   r¢  r²  Úoutputss   &&&&&&&&&&,   r0   rp   ÚStep3p7Model.forward  s'  € ð* Ò Ø ×5Ñ5Ô7¸	ÓBˆMàˆØÒ#Ø!×4Ñ4ØÐ2CÐQUð 5ó ç‰mð ô #ŸYšY ~¸1Ô=×@Ñ@À×AUÑAUÐWd×WjÑWjÓkˆNà!%×!:Ñ!:¸9ÐUcÓ!dÐØ)×8Ñ8Ð9KÓ\ˆMà×%Ò%ñ 
Øð
à'ð
ð *ð
ð &ð	
ð
 ,ð
ð  ð
ð ñ
ˆô *Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø .ô
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r*   rX  )r.   r/   s   "€r0   r1   rŒ  ´  s   ø‡ ‚ ñ Ñò r3   )r–  r’  r”  r�  r`  r  )	NNNNNNNNN)r?   r{   r|   r}   r[  r:   r   r   r©  r´  rp   rd  r�   r‚   rƒ   r„   s   @@r0   rŠ  rŠ  ´  sl   ù‡ € ð  Ð÷ó ð Ø÷)
ñ )
ó ó ð)
÷V"ð "ð. Ø÷0
ñ 0
ó ó ð0
÷y … r3   rŠ  zS
    Base class for Step3p7 causal language model (or autoregressive) outputs.
    c                   óH   a € ] tR tRt o RtRtRtRtRtRt	Rt
V 3R ltRtV tR# )ÚStep3p7CausalLMOutputWithPastiC  a  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size (batch_size * num_patches, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
Nc                ó:  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   S[ P                  R,          ;R&   # )r+   NÚlossÚlogitsr<  r’   rg  r†  )rL   r2  r   rK   )r.   r/   s   "€r0   r1   Ú*Step3p7CausalLMOutputWithPast.__annotate__C  s˜   ø‡ ‚ ñ, ×
Ñ
˜dÕ
"Ñ)ñ- ñ. ×Ñ Õ$Ñ+ñ/ ñ0 ˜T•\Ñ(ñ1 ñ2 ™×*Ñ*Õ+¨dÕ2Ñ9ñ3 ñ4 ‘e×'Ñ'Õ(¨4Õ/Ñ6ñ5 ñ6 ×*Ñ*¨TÕ1Ñ8ò7 r3   rZ  )r?   r{   r|   r}   r~   rÁ  rÂ  r<  r’   rg  r†  rd  r�   r‚   rˆ  s   @r0   r¿  r¿  C  s2   ø‡ € ñð &*€DØ'+€FØ$(€OØ59€MØ26€JØ48Ð÷7 ƒ r3   r¿  c                   ó¾   a a€ ] tR tRt oRR/tV3R lV 3R lltV3R lR ltR t]]	RV3R	 lR
 ll4       4       t
]]	RV3R lR ll4       4       tV3R ltRtVtV ;t# )ÚStep3p7ForConditionalGenerationia  zlm_head.weightz(model.language_model.embed_tokens.weightc                ó    <€ V ^8„  d   QhRS[ /# r*   rX  )r.   r/   s   "€r0   r1   Ú,Step3p7ForConditionalGeneration.__annotate__f  s   ø€ ÷ ñ ™}ñ r3   c                óò   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  P                  VP                  P                  R R7      V n	        V P                  4        R# )Fr¶  N)r9   r:   rŠ  r:  rA   rŠ   r‘  rV   r`  Úlm_headrt  rŽ   s   &&€r0   r:   Ú(Step3p7ForConditionalGeneration.__init__f  sS   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ü—y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Ör3   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rÃ  )rA   r­   )r.   r/   s   "€r0   r1   rÇ  l  s   ø€ ÷ ñ ¡r§y¡yñ r3   c                ó   € V P                   # rz   )rÉ  r°  s   &r0   Úget_output_embeddingsÚ5Step3p7ForConditionalGeneration.get_output_embeddingsl  s   € Ø�|‰|Ðr3   c                ó   € \        R 4      h)z(Step3p7 does not use pack_image_features)ÚNotImplementedErrorr°  s   &r0   Úpack_image_featuresÚ3Step3p7ForConditionalGeneration.pack_image_featureso  s   € Ü!Ð"LÓMÐMr3   c                óÐ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          S[ P                  ,          R,          RS[S[,          RS[S[,          /# )r+   r1  r™  Nrš  rZ   rJ   )	rL   r2  r›  r›   rM   r   r   rK   r   )r.   r/   s   "€r0   r1   rÇ  t  sl   ø€ ÷ 
ñ 
á×'Ñ'ð
ñ "×-Ñ-°Õ4ð
ñ  ¡�9¡u§|¡|Õ3°dÕ:ð	
ñ
 Ñ+Õ,ð
ñ 
Ñ+Õ	+ñ
r3   c           	     óF   € V P                   P                  ! RRVRVRV/VB # )a¸  
pixel_values (`torch.FloatTensor` of shape `(batch_size, 3, height, width)`):
    The tensors corresponding to the global view input images.
pixel_values_local (`torch.FloatTensor` of shape `(total_patches, 3, height, width)`, *optional*):
    All local patches flattened across the batch, or `None` if no local views.
num_local_patches (`list[int]` or `torch.Tensor`, *optional*):
    Number of local patches per image, e.g. `[6, 0, 4]`.
r1  r™  rš  rZ  )r:  r©  )rF   r1  r™  rš  rZ   s   &&&&,r0   r©  Ú2Step3p7ForConditionalGeneration.get_image_featuresr  s?   € ð" �z‰z×,Ò,ñ 
Ø%ð
à1ð
ð 0ð
ð ñ	
ð 	
r3   c                óø  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          S[ P
                  ,          R,          RS[ P
                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[S[ P
                  ,          RS[S[	,          RS[
S[,          /# )r+   rk  Nr1  r™  rš  rª   rm   r<  rl  ÚlabelsrE  Úlogits_to_keeprZ   rJ   )rL   rF  r2  r›  r›   rM   r   rG  r   r   rK   r¿  )r.   r/   s   "€r0   r1   rÇ  Œ  s  ø€ ÷ 7
ñ 7
á×#Ñ# dÕ*ð7
ñ ×'Ñ'¨$Õ.ð7
ñ "×-Ñ-°Õ4ð	7
ñ
  ¡�9¡u§|¡|Õ3°dÕ:ð7
ñ Ÿ™ tÕ+ð7
ñ ×&Ñ&¨Õ-ð7
ñ  �ð7
ñ ×(Ñ(¨4Õ/ð7
ñ × Ñ  4Õ'ð7
ñ ˜$•;ð7
ñ ™eŸl™lÕ*ð7
ñ Ñ+Õ,ð7
ñ 
Ñ.Õ	.ñ7
r3   c                óÊ  € V P                   ! RRVRVRVRVRVRVRVRVR	V
/	VB pV^ ,          p\        V\        4      '       d   \        V) R
4      MTpVRVR3,          pV P	                  V4      pR
pV	e7   V P
                  ! RRVRV	RV P                  P                  P                  /VB p\        VVVP                  VP                  VP                  VP                  R7      # )r¸  rk  r1  r™  rš  rª   rm   r<  rl  rE  Nr�   rÂ  r×  r`  )rÁ  rÂ  r<  r’   rg  r†  rZ  )r:  rf   r›   ÚslicerÉ  Úloss_functionr,   r‘  r`  r¿  r<  r’   rg  r†  )rF   rk  r1  r™  rš  rª   rm   r<  rl  r×  rE  rØ  rZ   r»  r’   Úslice_indicesrÂ  rÁ  s   &&&&&&&&&&&&,     r0   rp   Ú'Step3p7ForConditionalGeneration.forwardŠ  sB  € ð. —*’*ñ 
Øð
à%ð
ð  2ð
ð 0ð	
ð
 *ð
ð &ð
ð ,ð
ð (ð
ð  ð
ð ñ
ˆð   �
ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ% a¨¸Ð&9Õ:ˆØ—‘˜mÓ,ˆàˆØÒØ×%Ò%ñ Øðàðð  Ÿ;™;×2Ñ2×=Ñ=ðð ñ	ˆDô -ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r*   rX  )r.   r/   s   "€r0   r1   rÇ  a  s   ø‡ ‚ ñ Ñò r3   )rÉ  r:  r  )NNNNNNNNNNr   )r?   r{   r|   r}   Ú_tied_weights_keysr:   rÍ  rÑ  r   r   r©  rp   rd  r�   r‚   rƒ   r„   s   @@r0   rÅ  rÅ  a  su   ù‡ € à*Ð,VÐWÐ÷ó ÷ð òNð Ø÷
ñ 
ó ó ð
ð, Ø÷7
ñ 7
ó ó ð7
÷W … r3   rÅ  )rá   )r  )rÅ  rŠ  r9  rY  rf  )UÚcollections.abcr   Údataclassesr   rL   Útorch.nnrA   Útorch.nn.functionalr´   rß  Ú r   rG  Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úvision_utilsr   Úconfiguration_step3p7r    r!   r"   r­   r$   r†   r¤   r¼   rÀ   rÌ   rÎ   rî   r  r9  rf  rJ  rT  r´  rM  rQ  r  r  r   r;  rW  rY  r„  rŠ  r¿  rÅ  Ú__all__rZ  r3   r0   Ú<module>rõ     sV  ðõ* %Ý !ã Ý ß Ð å &Ý !ß .Ý )ß RÝ 9ß qÓ qß Kß FÝ &ß lÕ lÝ 0ß GÝ 5Ý 3ß XÑ Xô>5 2§9¡9ô >5ôB�r—y‘yô õ	U÷%ò2(õô88˜RŸY™Yô 88ôv"Ð :ô "ôJ:˜bŸi™iô :ðz ô('˜_ó ('ó ð('ðV ô/[Ð/ó /[ó ð/[ôdG<˜RŸY™Yô G<ôT=�R—Y‘Yô =ô,)�—‘ô )ô($�R—Y‘Yô $ôN9˜Ÿ	™	ô 9ô*˜BŸI™Iô ô6#ôLM)�r—y‘yô M)ô`1Ð4ô 1õh
ð ôe
Ð-ó e
ó ðe
ñP ðôð
 ô9Ð!8ó 9ó óð9ñ ðôô
G
Ð)ó G
óð
G
ñT ðôð
 ô9 Kó 9ó óð9ð0 ôa
Ð&<¸oó a
ó ða
òH‚r3   