+
    œ°³jI  ã                   ól  € R t ^ RIHt ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHt ^RIHtHt ^R	IHt ^R
IHtHtHt ^RIHt ^RIHtHtHt ^RIHt ^RIH t  ]PB                  ! ]"4      t# ! R R]PH                  4      t%R t&]! R4      R%R l4       t'R&R R llt(]	! ]'4       ! R R]PH                  4      4       t) ! R R]PH                  4      t* ! R R]PH                  4      t+ ! R R]4      t, ! R R]PH                  4      t-] ! R  R!]4      4       t.R" t/] ! R# R$].4      4       t0R$R!.t1R# )'zPyTorch Pixtral model.)ÚCallableN)Únn)ÚACT2FN)Úuse_kernel_forward_from_hubÚuse_kernelized_func)ÚGradientCheckpointingLayer)ÚBaseModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Údeprecate_kwarg)Úis_flash_attention_requestedÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputs©ÚPixtralVisionConfigc                   óÎ   a a€ ] tR t^&t oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       R	 4       tR
 tRtVtV ;t# )ÚPixtralVisionRotaryEmbeddingzí
Simple axial 2D rope with same freqs used for H and W grids. The freqs are
pre-computed using `head-dim//4` which is later used to concat H and W positions.
The final angles rotate over the whole head dim, no partial rotation involved.
Údevicez5.18)Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úm/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/pixtral/modeling_pixtral.pyÚ__annotate__Ú)PixtralVisionRotaryEmbedding.__annotate__.   s   ø€ ÷ Oñ OÑ2ñ Oó    c                óÎ  <€ \         SV `  4        Wn        V P                  P                  R ,          V n        V P
                  pV P                  R8w  d/   \        V P                  P                   RV P                   24      hV! V P                  V4      w  q@n	        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )Ú	rope_typeÚaxialz) supports only axial rope, but requested F)Ú
persistentN)ÚsuperÚ__init__r   Úrope_parametersr%   Úcompute_axial_rope_parametersÚ
ValueErrorÚ	__class__Ú__name__Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr   r   Úrope_init_fnr1   r-   s   &&&  €r    r)   Ú%PixtralVisionRotaryEmbedding.__init__-   s¬   ø€ ä‰ÑÔØŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!CÑ!CˆØ�>‰>˜WÔ$Ü §¡× 7Ñ 7Ð8Ð8aÐbf×bpÑbpÐaqÐrÓsÐsÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr#   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r   r   Úreturn)r   ÚtupleÚtorchÚTensorÚfloat)r   r   s   "€r    r!   r"   =   s+   ø€ ÷ 8ñ 8Ù#ð8á	‰u�|‰|™UÐ"Õ	#ñ8r#   c           	     ó|  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          p\        P                  ! VR,          VR,          .4      pVP                  V4      V3# )a{  
Computes the inverse frequencies according to the axial RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?©Údtype:é    Nr   :é   Nr   )	r*   ÚgetattrÚhidden_sizeÚnum_attention_headsr:   Úaranger<   ÚcatÚto)r   r   ÚkwargsÚbaseÚdimÚattention_factorr1   Úinv_freq_2ds   &&,     r    r+   Ú:PixtralVisionRotaryEmbedding.compute_axial_rope_parameters;   s™   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆÜ—i’i ¨$¥°¸$µÐ @ÓAˆà�~‰~˜fÓ%Ð'7Ð7Ð7r#   c                óÒ  € V P                   R,          P                  4       pVR,          P                  4       p\        VP                  P                  \
        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        WC,          pVP                  4       V P                  ,          pVP                  4       V P                  ,          pR R R 4       V P                  X4      pV P                  X4      pVP                  VP                  4      VP                  VP                  4      3#   + '       g   i     Lh; i)NÚmpsÚcpuF)Údevice_typeÚenabled)N.).N)r1   r<   Ú
isinstancer   ÚtypeÚstrr   Úcosr/   ÚsinÚrecomposition_frequenciesrI   rA   )	r4   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrS   ÚfreqsrX   rY   s	   &&&      r    ÚforwardÚ$PixtralVisionRotaryEmbedding.forwardS   sø   € à ŸM™M¨)Õ4×:Ñ:Ó<ÐØ ,¨YÕ 7× =Ñ =Ó ?Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ)Õ=ˆEØ—)‘)“+ × 6Ñ 6Õ6ˆCØ—)‘)“+ × 6Ñ 6Õ6ˆC÷ Dð
 ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØ�v‰v�a—g‘g‹ §¡ q§w¡w£Ð/Ð/÷ D×Cús   Â+AEÅE&	c                ó°   € R \        VP                  ^RR7      4       4       w  r#\        P                  ! W#.RR7      p\        P                  ! WD.RR7      # )zM
Recompose the frequencies into the final spatial layout used per each grid.
c              3   óJ   "  € T F  w  rVR V^,          3,          x € K  	  R# 5i)ºNNNN© )Ú.0ÚiÚms   &  r    Ú	<genexpr>ÚIPixtralVisionRotaryEmbedding.recomposition_frequencies.<locals>.<genexpr>g   s"   é € ÐSÑ2R©$¨!˜!˜A˜q 1�u˜HŸ+š+Ó2Rùs   ‚!#©rL   éÿÿÿÿ)Ú	enumerateÚchunkr:   rH   )r4   ÚfreqÚfreq_hÚfreq_wÚfreq_hws   &&   r    rZ   Ú6PixtralVisionRotaryEmbedding.recomposition_frequenciesb   sJ   € ñ
 T´)¸D¿J¹JÀqÈb¸JÓ<QÔ2RÓS‰ˆÜ—)’)˜VÐ,°"Ô5ˆÜ�yŠy˜'Ð+°Ô4Ð4r#   )r/   r   r1   r3   r%   ©N)r.   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r)   Ústaticmethodr+   r:   Úno_gradr`   rZ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r-   r   s   @@r    r   r   &   sy   ù‡ € ññ �X vÔ.÷Oô Oó /ðOð Ù�X vÔ.÷8ñ 8ó /ó ð8ð, ‡]‚]ƒ_ñ0ó ð0÷5ò 5r#   r   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nrk   rl   )Úshaper:   rH   )r[   Úx1Úx2s   &  r    Úrotate_halfrƒ   m   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r#   Úrotary_pos_embc                óÈ   € VP                  V4      pVP                  V4      pW,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWV3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerƒ   )ÚqÚkrX   rY   Úunsqueeze_dimÚq_embedÚk_embeds   &&&&&  r    Úapply_rotary_pos_embrŒ   t   sS   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr#   c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropout)r   ÚModuler:   r;   r<   )r   s   "r    r!   r!   �   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r#   c                 óÎ  € \         P                  ! WP                  RR4      4      V,          pVe	   W„,           p\        P                  P                  VR\         P                  R7      P                  VP                  4      p\        P                  P                  W†V P                  R7      p\         P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )rC   )rL   rA   )ÚpÚtrainingrl   éþÿÿÿ)r:   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32rI   rA   r”   r˜   Ú
contiguous)
rŽ   r�   r�   r‘   r’   r“   r”   rJ   Úattn_weightsÚattn_outputs
   &&&&&&&,  r    Úeager_attention_forwardr¢   �   s°   € ô —<’< §}¡}°R¸Ó'<Ó=ÀÕG€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r#   c                   óL   a a€ ] tR t^§t oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚPixtralAttentionzA
Multi-headed attention compatible with ALL_ATTENTION_FUNCTIONS.
c                ó¶  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        R V n        V P                  R,          V n	        R V n        VP                  V n        \        P                  ! V P                  V P                  R R7      V n        \        P                  ! V P                  V P                  R R7      V n        \        P                  ! V P                  V P                  R R7      V n        \        P                  ! V P                  V P                  R R7      V n        R# )F©ÚbiasNg      à¿)r(   r)   r   rE   Ú	embed_dimrF   Ú	num_headsr?   Ú	is_causalr“   Úattention_dropoutr”   r   ÚLinearÚk_projÚv_projÚq_projÚo_proj©r4   r   r-   s   &&€r    r)   ÚPixtralAttention.__init__­   sÜ   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Õ8ˆŒØˆŒà—}‘} dÕ*ˆŒØˆŒà×/Ñ/ˆŒä—i’i §¡°·±ÀUÔKˆŒÜ—i’i §¡°·±ÀUÔKˆŒÜ—i’i §¡°·±ÀUÔKˆŒÜ—i’i §¡°·±ÀUÔKˆŽr#   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r   Úhidden_statesr’   NÚposition_embeddingsrJ   r8   ©r:   r;   r9   r   r   )r   r   s   "€r    r!   ÚPixtralAttention.__annotate__¿   s~   ø€ ÷ ()ñ ()á—|‘|ð()ñ Ÿ™ tÕ+ð()ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	()ñ
 Ñ+Õ,ð()ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ()r#   c                ó<  € VP                  4       w  rVpV P                  V4      pV P                  V4      p	V P                  V4      p
VP	                  WVV P
                  V P                  4      P                  ^^4      pV	P	                  WVV P
                  V P                  4      P                  ^^4      p	V
P	                  WVV P
                  V P                  4      P                  ^^4      p
Vw  r¼\        W‰W¼^ R7      w  r‰\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                  RV P                   /VB w  rïVP#                  WVR4      P%                  4       pV P'                  V4      pWï3# )z#Input shape: Batch x Time x Channel)r‰   r”   ç        r“   rl   )Úsizer¯   r­   r®   Úviewr©   r?   r›   rŒ   r	   Úget_interfacer   Ú_attn_implementationr¢   r˜   r”   r“   ÚreshaperŸ   r°   )r4   r´   r’   rµ   rJ   Ú
batch_sizeÚpatchesÚ_Úquery_statesÚ
key_statesÚvalue_statesrX   rY   Úattention_interfacer¡   r    s   &&&&,           r    r`   ÚPixtralAttention.forward¿   s}  € ð "/×!3Ñ!3Ó!5Ñˆ
˜Qà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆØ—_‘_ Z¸$¿.¹.È$Ï-É-ÓX×bÑbÐcdÐfgÓhˆ
Ø#×(Ñ(¨¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆà&‰ˆÜ#7¸ÐRUÐjkÔ#lÑ ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨*¸rÓB×MÑMÓOˆØ—k‘k +Ó.ˆàÐ(Ð(r#   )r   r”   r¨   r?   rª   r­   r©   r°   r¯   r“   r®   ©NN)
r.   ru   rv   rw   rx   r)   r`   r{   r|   r}   r~   s   @@r    r¤   r¤   §   s   ù‡ € ñõL÷$()÷ ()ò ()r#   r¤   c                   ó8   a a€ ] tR t^ët oV 3R ltR tRtVtV ;t# )Ú
PixtralMLPc                óØ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n        \
        P                  ! V P                  V P                  R R7      V n	        \        VP                  ,          V n        R# )Fr¦   N)r(   r)   r   rE   Úintermediate_sizer   r¬   Ú	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnr±   s   &&€r    r)   ÚPixtralMLP.__init__ì   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Õ/ˆŽr#   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# rt   )rÎ   rÐ   rÌ   rÍ   )r4   r[   rÎ   s   && r    r`   ÚPixtralMLP.forwardö   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr#   )rÐ   r   rÎ   rÌ   rE   rË   rÍ   ©	r.   ru   rv   rw   r)   r`   r{   r|   r}   r~   s   @@r    rÉ   rÉ   ë   s   ù‡ € õ0÷ò r#   rÉ   c                   óZ   a a€ ] tR t^üt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )ÚPixtralRMSNormc                ó$   <€ V ^8„  d   QhRS[ RR/# )r   Úepsr8   N)r<   )r   r   s   "€r    r!   ÚPixtralRMSNorm.__annotate__ý   s   ø€ ÷ $ñ $©ð $¸$ñ $r#   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z-
PixtralRMSNorm is equivalent to T5LayerNorm
N)r(   r)   r   Ú	Parameterr:   ÚonesÚweightÚvariance_epsilon)r4   rE   rØ   r-   s   &&&€r    r)   ÚPixtralRMSNorm.__init__ý   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r´   r8   )r:   r;   )r   r   s   "€r    r!   rÙ     s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r#   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )r   T)Úkeepdimrl   )	rA   rI   r:   rž   ÚpowÚmeanÚrsqrtrÞ   rÝ   )r4   r´   Úinput_dtypeÚvariances   &&  r    r`   ÚPixtralRMSNorm.forward  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r#   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)r9   rÝ   r€   rÞ   ©r4   s   &r    Ú
extra_reprÚPixtralRMSNorm.extra_repr  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr#   )rÞ   rÝ   )g�íµ ÷Æ°>)
r.   ru   rv   rw   r)   r`   rë   r{   r|   r}   r~   s   @@r    rÖ   rÖ   ü   s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr#   rÖ   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚPixtralAttentionLayeri  c                óÔ   <€ \         SV `  4        \        VP                  R R7      V n        \        V4      V n        \        V4      V n        \        VP                  R R7      V n	        R# )çñhãˆµøä>©rØ   N)
r(   r)   rÖ   rE   Úattention_normrÉ   Úfeed_forwardr¤   Ú	attentionÚffn_normr±   s   &&€r    r)   ÚPixtralAttentionLayer.__init__  sP   ø€ Ü‰ÑÔÜ,¨V×-?Ñ-?ÀTÔJˆÔÜ& vÓ.ˆÔÜ)¨&Ó1ˆŒÜ& v×'9Ñ'9¸tÔDˆŽr#   c          
      óÐ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[ P                  /# )r   r´   r’   rµ   NrJ   r8   r¶   )r   r   s   "€r    r!   Ú"PixtralAttentionLayer.__annotate__  sf   ø€ ÷ ñ á—|‘|ðñ Ÿ™ðñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	ñ
 Ñ+Õ,ðñ 
�‰ñr#   c           	     óÈ   € TpV P                  V4      pV P                  ! RRVRVRV/VB w  rWQ,           pTpV P                  V4      pV P                  V4      pWQ,           pV# )a"  
Args:
    hidden_states (`torch.FloatTensor`):
        Input to the layer of shape `(batch, seq_len, embed_dim)`.
    attention_mask (`torch.FloatTensor`):
        Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
r´   r’   rµ   re   )rò   rô   rõ   ró   )r4   r´   r’   rµ   rJ   ÚresidualrÁ   s   &&&&,  r    r`   ÚPixtralAttentionLayer.forward  s‰   € ð !ˆà×+Ñ+¨MÓ:ˆØŸ>š>ñ 
Ø'ð
à)ð
ð !4ð
ð ñ	
Ñˆð !Õ0ˆà ˆØŸ™ mÓ4ˆØ×)Ñ)¨-Ó8ˆØ Õ0ˆàÐr#   )rô   rò   ró   rõ   rt   rÔ   r~   s   @@r    rî   rî     s   ù‡ € õE÷÷ ò r#   rî   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚPixtralTransformeri9  c                ó  <€ \         SV `  4        Wn        \        P                  P                  4       V n        \        VP                  4       F'  pV P                  P                  \        V4      4       K)  	  R V n        R# )FN)r(   r)   r   r:   r   Ú
ModuleListÚlayersÚrangeÚnum_hidden_layersÚappendrî   Úgradient_checkpointing)r4   r   rÁ   r-   s   && €r    r)   ÚPixtralTransformer.__init__:  s\   ø€ Ü‰ÑÔØŒÜ—h‘h×)Ñ)Ó+ˆŒÜ�v×/Ñ/Ö0ˆAØ�K‰K×ÑÔ4°VÓ<Ö=ñ 1à&+ˆÖ#r#   c          	      óÀ   <€ V ^8„  d   QhRS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[S[,          RS[S[,          /# )r   r’   Nrµ   rJ   r8   )r:   r;   r9   r   r   r   )r   r   s   "€r    r!   ÚPixtralTransformer.__annotate__B  sf   ø€ ÷ @ñ @ñ Ÿ™ tÕ+ð@ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð	@ñ
 Ñ+Õ,ð@ñ 
‘Õ	 ñ@r#   c                ó\   € TpV P                    F  pV! VV3RV/VB pK  	  \        VR7      # )a  
Args:
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Embeddings which serve as input to the Transformer.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
rµ   )Úlast_hidden_state)r   r   )r4   Úinputs_embedsr’   rµ   rJ   r´   Úencoder_layers   &&&&,  r    r`   ÚPixtralTransformer.forwardB  sH   € ð& &ˆØ!Ÿ[œ[ˆMÙ)ØØñð %8ðð ñ	ŠMñ )ô °Ô?Ð?r#   )r   r  r   rÇ   rÔ   r~   s   @@r    rý   rý   9  s   ù‡ € õ,÷@÷ @ò @r#   rý   c                   ó^   a € ] tR tRt o RtRtRtRtRtRt	Rt
RtR.tR]R]/tV 3R ltR	tV tR
# )ÚPixtralPreTrainedModelia  ÚmodelÚpixel_valuesTrî   r´   Ú
attentionsc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r   r   )r   r   s   "€r    r!   Ú#PixtralPreTrainedModel.__annotate__a  s   ø‡ ‚ áÑò r#   re   N)Úimage)r.   ru   rv   rw   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_no_split_modulesrî   r¤   Ú_can_record_outputsÚ__annotate_func__r{   r|   )r   s   @r    r  r  a  sY   ø‡ € ð  ÐØ$€OØ!ÐØ&*Ð#Ø"&ÐØÐØ€NØÐØ0Ð1ÐàÐ.ØÐ&ðÐ÷ ƒ r#   r  c                 ó  € VP                   pVP                  pVP                  ^,          p\        P                  ! V4      P
                  p\        P                  ! WD3WRVR7      p\        P                  ! V 4      P                  R4      p\        P                  ! ^ .V RR ,           4      P                  R4      p\        W‡4       F  w  rš^ WiV
1Wš13&   K  	  VR,          P                  VP                  ^ ,          ^RR4      pV# )rC   )Ú
fill_valuerA   r   Nrl   )NNrd   rd   )rA   r   r€   r:   ÚfinfoÚminÚfullÚtensorÚcumsumÚzipÚexpand)Úpatch_embeds_listr%  rA   r   Úseq_lenÚd_minÚcausal_maskÚblock_end_idxÚblock_start_idxÚstartÚends   &&         r    Úgenerate_block_attention_maskr1  s  sÝ   € Ø�L‰L€EØ�]‰]€FØ�l‰l˜1�o€GÜ�KŠK˜Ó×"Ñ"€EÜ—*’*˜gÐ/¸EÐW]Ô^€Kä—L’LÐ!2Ó3×:Ñ:¸2Ó>€MÜ—l’l A 3Ð):¸3¸BÐ)?Õ#?Ó@×GÑGÈÓK€OÜ˜/Ö9‰
ˆØ,-ˆ˜#�I˜u˜yÐ(Ó)ñ :ð Ð.Õ/×6Ñ6°v·|±|ÀAµÈÈ2ÈrÓR€KØÐr#   c                   óp   a a€ ] tR tRt oRtV 3R ltR t]]]	RV3R lR ll4       4       4       t
RtVtV ;t# )	ÚPixtralVisionModeliƒ  Úvision_encoderc                ó|  <€ \         SV `  V4       Wn        \        P                  ! VP
                  VP                  VP                  VP                  R R7      V n        VP                  V n        \        VP                  RR7      V n
        \        V4      V n        \        V4      V n        V P                  4        R# )F)Úin_channelsÚout_channelsÚkernel_sizeÚstrider§   rð   rñ   N)r(   r)   r   r   ÚConv2dÚnum_channelsrE   Ú
patch_sizeÚ
patch_convrÖ   Úln_prerý   Útransformerr   Úpatch_positional_embeddingÚ	post_initr±   s   &&€r    r)   ÚPixtralVisionModel.__init__‡  s“   ø€ Ü‰Ñ˜Ô ØŒÜŸ)š)Ø×+Ñ+Ø×+Ñ+Ø×)Ñ)Ø×$Ñ$Øô
ˆŒð !×+Ñ+ˆŒÜ$ V×%7Ñ%7¸TÔBˆŒÜ-¨fÓ5ˆÔÜ*FÀvÓ*NˆÔ'à�‰Ör#   c                ó   € V P                   # rt   )r=  rê   s   &r    Úget_input_embeddingsÚ'PixtralVisionModel.get_input_embeddings˜  s   € Ø�‰Ðr#   c          	      óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[,          /# )r   r  Úimage_sizesNrJ   r8   )r:   r;   r   r   r9   r   )r   r   s   "€r    r!   ÚPixtralVisionModel.__annotate__ž  sK   ø€ ÷ 0
ñ 0
á—l‘lð0
ñ —\‘\ DÕ(ð0
ñ Ñ+Õ,ð	0
ñ
 
‘Õ	 ñ0
r#   c           
     ó<  € Vf   VP                   w  rErgWg3.V,          pV P                  P                  P                  pV P                  VP	                  VR7      4      p	\        W’4       U
Uu. uFE  w  r«V
RR V^ ,          V P                  ,          1R V^,          V P                  ,          13,          NKG  	  pp
p\        P                  ! V Uu. uF  qÝP                  ^4      P                  NK  	  up^ R7      P                  ^ 4      p	V P                  V	4      p	. pV FÆ  p\        P                  ! \        P                  ! VP                   R
,          VP                  R7      \        P                  ! VP                   R,          VP                  R7      RR7      w  ppVP!                  \        P"                  ! VP                  4       VP                  4       .RR7      4       KÈ  	  \        P                  ! V^ R7      pV P%                  Wž4      p\'        V P(                  4      '       d   R pMD\+        V Uu. uF-  qÝP                   R
,          VP                   R,          ,          NK/  	  upV	4      pV P,                  ! V	3RVRVR	V/VB # u upp
i u upi u upi )Nr@   .rk   )r   Úij)Úindexingr’   rµ   r\   r™   rl   )r€   r=  rÝ   rA   rI   r'  r<  r:   rH   ÚflattenÚTr†   r>  ÚmeshgridrG   r   r  Ústackr@  r   r   r1  r?  )r4   r  rG  rJ   r¿   rÁ   ÚheightÚwidthÚtarget_dtypeÚpatch_embedsÚembedrº   r)  r—   r\   ÚpatchÚhpos_idsÚwpos_idsrµ   r’   s   &&&,                r    r`   ÚPixtralVisionModel.forward›  sJ  € ð ÒØ+7×+=Ñ+=Ñ(ˆJ˜6Ø"˜?Ð+¨jÕ8ˆKð —‘×-Ñ-×3Ñ3ˆØ—‘ |§¡¸\ Ó'JÓKˆô  # <Ô=ô
á=‘�ð �#Ð5˜$˜q�' T§_¡_Õ4Ð5Ð7U¸$¸q½'ÀTÇ_Á_Õ:TÐ7UÐU×VÐVÙ=ð 	ñ 
ô —y’yÑ:KÓ!LÑ:K°Q§)¡)¨A£,§.¤.Ñ:KÑ!LÐRSÔT×^Ñ^Ð_`ÓaˆØ—{‘{ <Ó0ˆð ˆÛ&ˆEÜ!&§¢Ü—’˜UŸ[™[¨�_°U·\±\ÔBÜ—’˜UŸ[™[¨�_°U·\±\ÔBØô"ÑˆH�hð
 ×Ñ¤§¢¨X×-=Ñ-=Ó-?À×AQÑAQÓASÐ,TÐZ\Ô ]Ö^ñ 'ô —y’y °1Ô5ˆØ"×=Ñ=¸lÓYÐä'¨¯©×4Ò4à!‰Nä:Ù4EÓFÑ4E¨q—‘˜•˜qŸw™w r�{×*Ð*Ñ4EÑFÈóˆNð ×ÒØñ
à)ð
ð !4ð
ð &ð	
ð
 ñ
ð 	
ùó;
ùò "Mùò( Gs   Á.AJÃ#JÈ:3J)r   r>  r=  r@  r<  r?  rt   )r.   ru   rv   rw   r  r)   rD  r   r   r   r`   r{   r|   r}   r~   s   @@r    r3  r3  ƒ  sC   ù‡ € à(Ðõò"ð  ØØ÷0
ñ 0
ó ó ó  ÷0
ð 0
r#   r3  )rC   )r¹   )2rx   Úcollections.abcr   r:   r   Úactivationsr   Úintegrationsr   r   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr	   r
   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_pixtralr   Ú
get_loggerr.   Úloggerr•   r   rƒ   rŒ   r¢   r¤   rÉ   rÖ   rî   rý   r  r1  r3  Ú__all__re   r#   r    Ú<module>rh     s9  ðñ å $ã Ý å !ß LÝ 9Ý /ß FÝ &ß @Ñ @Ý 0ß eÑ eÝ 5Ý 6ð 
×	Ò	˜HÓ	%€ôC5 2§9¡9ô C5òN(ñ Ð-Ó.óó /ð÷4%ñ. Ð)Ó*ô?)�r—y‘yó ?)ó +ð?)ôF�—‘ô ô"J�R—Y‘Yô Jô(&Ð6ô &ôR%@˜Ÿ™ô %@ðP ô˜_ó ó ðò"ð  ôJ
Ð/ó J
ó ðJ
ðZ  Ð!9Ð
:‚r#   