+
    œ°³j\  ã                   ó–  € R t ^ RIt^ RIHtHt ^ RIt^ RIHt ^RIHt	 ^RI
Ht ^RIHt ^RIHt ^RIHtHtHt ^R	IHtHt ^R
IHt ^RIHtHtHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$ ]PJ                  ! ]&4      t' ! R R]PP                  4      t) ! R R]PP                  4      t*R(R R llt+ ! R R]PP                  4      t, ! R R]PP                  4      t- ! R R]4      t. ! R R]PP                  4      t/] ! R R]4      4       t0] ! R R ]04      4       t1]! R!R"7       ! R# R$]04      4       t2]! R%R"7       ! R& R']04      4       t3. R)Ot4R# )*zPyTorch ViT model.N)ÚCallableÚIterable)Únn)Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPoolingÚImageClassifierOutputÚMaskedImageModelingOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs©Ú	ViTConfigc                   óT   a a€ ] tR t^*t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚViTPatchEmbeddingszÜ
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/vit/modeling_vit.pyÚ__annotate__ÚViTPatchEmbeddings.__annotate__1   s   ø€ ÷ xñ x™yñ xó    c                óÎ  <€ \         SV `  4        VP                  pVP                  p\	        V\
        4      '       d   TMW"3p\	        V\
        4      '       d   TMW33pV^,          V^,          ,          V^ ,          V^ ,          ,          ,          V n        W n        W0n        VP                  V n        \        P                  ! VP                  VP                  W3R7      V n        R# )é   )Úkernel_sizeÚstrideN)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsr   ÚConv2dÚhidden_sizeÚ
projection)Úselfr   r*   r+   Ú	__class__s   &&  €r    r)   ÚViTPatchEmbeddings.__init__1   s³   ø€ Ü‰ÑÔØ×&Ñ&ˆ
Ø×&Ñ&ˆ
Ü#-¨j¼(×#CÒ#C‘ZÈ*ÐIaˆ
Ü#-¨j¼(×#CÒ#C‘ZÈ*ÐIaˆ
à& q�M¨Z¸­]Õ:¸zÈ!½}ÐPZÐ[\ÕP]Õ?]Õ^ˆÔØ$ŒØ$ŒØ"×/Ñ/ˆÔÜŸ)š) F×$7Ñ$7¸×9KÑ9KÐYcÔwˆŽr#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úpixel_valuesÚreturn©ÚtorchÚTensor)r   r   s   "€r    r!   r"   >   s'   ø€ ÷ Hñ H¡E§L¡Lð H±U·\±\ñ Hr#   c                óà   € VP                   ^,          pW P                  8w  d   \        RV P                   RV R24      hV P                  V4      P	                  ^4      P                  ^^4      # )r%   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got Ú.)Úshaper.   Ú
ValueErrorr1   ÚflattenÚ	transpose)r2   r6   r.   s   && r    ÚforwardÚViTPatchEmbeddings.forward>   ss   € Ø#×)Ñ)¨!Õ,ˆØ×,Ñ,Ô,ÜðØ!×.Ñ.Ð/¨y¸¸ÀaðIóð ð �‰˜|Ó,×4Ñ4°QÓ7×AÑAÀ!ÀQÓGÐGr#   )r*   r.   r-   r+   r1   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r)   rA   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r3   r   s   @@r    r   r   *   s(   ù‡ € ñ÷xó x÷H÷ Hð Hr#   r   c                   ón   a a€ ] tR t^Ht oRtR	V3R lV 3R llltV3R lR ltR
V3R lR lltRtVt	V ;t
# )ÚViTEmbeddingszZ
Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r   Úuse_mask_token©r   Úbool)r   r   s   "€r    r!   ÚViTEmbeddings.__annotate__M   s   ø€ ÷ 
;ñ 
;™yð 
;¹$ñ 
;r#   c                ó–  <€ \         SV `  4        \        P                  ! \        P
                  ! ^^VP                  4      4      V n        V'       d7   \        P                  ! \        P                  ! ^^VP                  4      4      MRV n	        \        V4      V n        V P                  P                  p\        P                  ! \        P
                  ! ^V^,           VP                  4      4      V n        \        P                  ! VP                  4      V n        VP"                  V n        V P                  P$                  V n        R# )r%   N)r(   r)   r   Ú	Parameterr9   Úrandnr0   Ú	cls_tokenÚzerosÚ
mask_tokenr   Úpatch_embeddingsr-   Úposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutr+   r*   )r2   r   rO   r-   r3   s   &&& €r    r)   ÚViTEmbeddings.__init__M   sÖ   ø€ Ü‰ÑÔäŸš¤e§k¢k°!°Q¸×8JÑ8JÓ&KÓLˆŒßQ_œ"Ÿ,š,¤u§{¢{°1°a¸×9KÑ9KÓ'LÔMÐeiˆŒÜ 2°6Ó :ˆÔØ×+Ñ+×7Ñ7ˆÜ#%§<¢<´·²¸A¸{ÈQ½ÐPV×PbÑPbÓ0cÓ#dˆÔ Ü—z’z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØ×/Ñ/×:Ñ:ˆŽr#   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r   Ú
embeddingsÚheightÚwidthr7   )r9   r:   Úint)r   r   s   "€r    r!   rR   Y   s;   ø€ ÷ &Dñ &D±5·<±<ð &DÉð &DÑUXð &DÑ]b×]iÑ]iñ &Dr#   c                óæ  € VP                   ^,          ^,
          pV P                  P                   ^,          ^,
          p\        P                  P	                  4       '       g   WE8X  d   W#8X  d   V P                  # V P                  R,          pV P                  R,          pVP                   R,          pW P
                  ,          p	W0P
                  ,          p
\        VR,          4      pVP                  ^W»V4      pVP                  ^ ^^^4      p\        P                  P                  VWš3RRR7      pVP                  ^ ^^^4      P                  ^RV4      p\        P                  ! Wg3^R7      # )	aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
ç      à?ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)ºNNN:Nr%   N©rl   :r%   NNéÿÿÿÿ)r=   rZ   r9   ÚjitÚ
is_tracingr+   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r2   r`   ra   rb   r-   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrk   Ú
new_heightÚ	new_widthÚsqrt_num_positionss   &&&&        r    Úinterpolate_pos_encodingÚ&ViTEmbeddings.interpolate_pos_encodingY   sE  € ð !×&Ñ& qÕ)¨AÕ-ˆØ×0Ñ0×6Ñ6°qÕ9¸AÕ=ˆô �y‰y×#Ñ#×%Ò%¨+Ô*FÈ6Ì?Ø×+Ñ+Ð+à×2Ñ2°5Õ9ˆØ×2Ñ2°5Õ9ˆà×Ñ˜rÕ"ˆàŸ™Õ.ˆ
ØŸ_™_Õ,ˆ	ä& }°cÕ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÔCÐCr#   c                ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[ P                  /# )r   r6   Úbool_masked_posNr}   r7   )r9   r:   Ú
BoolTensorrQ   )r   r   s   "€r    r!   rR   �   sF   ø€ ÷  ñ  á—l‘lð ñ ×)Ñ)¨DÕ0ð ñ #'ð	 ñ
 
�‰ñ r#   c                óê  € VP                   w  rErgV P                  V4      pVem   VP                   ^,          p	V P                  P                  WIR4      p
VP	                  R4      P                  V
4      pVRV,
          ,          W«,          ,           pV P                  P                  VRR4      p\        P                  ! WÈ3^R7      pV'       d   W€P                  W†V4      ,           pMzW`P                  ^ ,          8w  g   WpP                  ^,          8w  d;   \        RV RV RV P                  ^ ,           RV P                  ^,           R2	4      hW€P                  ,           pV P                  V4      pV# )Ng      ð?rj   zInput image size (Ú*z) doesn't match model (z).rn   )r=   rY   rX   ÚexpandÚ	unsqueezeÚtype_asrV   r9   rv   r}   r*   r>   rZ   r]   )r2   r6   r€   r}   Ú
batch_sizer.   ra   rb   r`   Ú
seq_lengthÚmask_tokensÚmaskÚ
cls_tokenss   &&&&         r    rA   ÚViTEmbeddings.forward�   sN  € ð 3?×2DÑ2DÑ/ˆ
 &Ø×*Ñ*¨<Ó8ˆ
àÒ&Ø#×)Ñ)¨!Õ,ˆJØŸ/™/×0Ñ0°ÈÓLˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¥zÕ2°[Õ5GÕGˆJð —^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ü—Y’Y 
Ð7¸QÔ?ˆ
ç#Ø#×&CÑ&CÀJÐX]Ó&^Õ^‰JàŸ™¨Õ+Ô+¨u¿¹ÈÕ8JÔ/JÜ Ø(¨¨°°%°ð 9ØŸ™¨Õ+Ð,¨A¨d¯o©o¸aÕ.@Ð-AÀðEóð ð $×&>Ñ&>Õ>ˆJà—\‘\ *Ó-ˆ
àÐr#   )rV   r]   r*   rX   rY   r+   rZ   )F)NF)rC   rD   rE   rF   rG   r)   r}   rA   rH   rI   rJ   rK   s   @@r    rM   rM   H   s0   ù‡ € ñ÷
;õ 
;÷&Dð &D÷P ÷  ò  r#   rM   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingr]   Úkwargs)r   ÚModuler9   r:   Úfloatr   r   )r   s   "r    r!   r!   ¤   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%r#   c                 ó  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VR\        P                  R7      P                  VP                  4      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N)rk   Údtype)ÚpÚtrainingrn   ç      à¿)rg   r9   Úmatmulr@   r   rs   ÚsoftmaxÚfloat32Útor˜   r]   rš   Ú
contiguous)
rŽ   r�   r�   r‘   r’   r“   r]   r”   Úattn_weightsÚattn_outputs
   &&&&&&&,  r    Úeager_attention_forwardr£   ¤   sÉ   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r#   c                   óT   a a€ ] tR t^Àt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚViTAttentionc                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   ÚViTAttention.__annotate__Á   s   ø€ ÷ kñ k™yñ kr#   c                ó`  <€ \         SV `  4        Wn        VP                  V n        \	        VR VP
                  VP                  ,          4      V n        VP                  V n        V P                  R,          V n	        RV n
        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  RR7      V n        R# )Úhead_dimF)ÚbiasTNr›   )r(   r)   r   Únum_attention_headsÚgetattrr0   r©   Úattention_probs_dropout_probÚattention_dropoutr“   Ú	is_causalr   ÚLinearÚqkv_biasÚq_projÚk_projÚv_projÚo_proj©r2   r   r3   s   &&€r    r)   ÚViTAttention.__init__Á   s*  ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ü ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ!'×!DÑ!DˆÔØ—}‘} dÕ*ˆŒØˆŒä—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ðek×etÑetÔuˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ðek×etÑetÔuˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ðek×etÑetÔuˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^ÐeiÔjˆŽr#   c          
      ó¶   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# ©r   Úhidden_statesr’   Nr”   r7   )r9   r:   r   r   Útuple)r   r   s   "€r    r!   r§   Ï   sX   ø€ ÷ )ñ )á—|‘|ð)ñ Ÿ™ tÕ+ð)ñ Ñ+Õ,ð	)ñ
 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ)r#   c                ó¨  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p\        P                  ! V P                  P                  \        4      p	T	! V VVVV3RV P                  '       g   RMV P                  RV P                  /VB w  r«V
P                  ! . VORN5!  P!                  4       p
V P#                  V
4      p
W«3# )Nr]   ç        r“   rn   )r=   r©   r²   ru   r@   r³   r´   r   Úget_interfacer   Ú_attn_implementationr£   rš   r®   r“   rq   r    rµ   )r2   rº   r’   r”   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer¢   r¡   s   &&&,        r    rA   ÚViTAttention.forwardÏ   sN  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆàÐ(Ð(r#   )
r®   r   r©   r¯   r³   r«   rµ   r²   r“   r´   ©N©	rC   rD   rE   rF   r)   rA   rH   rI   rJ   rK   s   @@r    r¥   r¥   À   s    ù‡ € ÷kó k÷)÷ )ò )r#   r¥   c                   óP   a a€ ] tR t^ñt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚViTMLPc                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   ÚViTMLP.__annotate__ò   s   ø€ ÷ Kñ K™yñ Kr#   c                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # rÇ   )r(   r)   r   r   Ú
hidden_actÚactivation_fnr   r°   r0   Úintermediate_sizeÚfc1Úfc2r¶   s   &&€r    r)   ÚViTMLP.__init__ò   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rº   r7   r8   )r   r   s   "€r    r!   rÌ   ù   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r#   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rÇ   )rÑ   rÏ   rÒ   )r2   rº   s   &&r    rA   ÚViTMLP.forwardù   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆàÐr#   )rÏ   r   rÑ   rÒ   rÈ   rK   s   @@r    rÊ   rÊ   ñ   s    ù‡ € ÷Kó K÷÷ ð r#   rÊ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚViTLayeri  c                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   ÚViTLayer.__annotate__  s   ø€ ÷ >ñ >™yñ >r#   c                ór  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R 7      V n        \        P
                  ! VP                  VP                  R 7      V n	        \        V4      V n        \        P                  ! VP                  4      V n        R# )©ÚepsN)r(   r)   r¥   Ú	attentionr   Ú	LayerNormr0   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterrÊ   Úmlpr[   r\   r]   r¶   s   &&€r    r)   ÚViTLayer.__init__  sy   ø€ Ü‰ÑÔÜ% fÓ-ˆŒÜ "§¢¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜ˜&“>ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr#   c                óŒ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          RS[ P                  /# r¹   )r9   r:   r   r   )r   r   s   "€r    r!   rÛ   
  sJ   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ Ñ+Õ,ð	ñ
 
�‰ñr#   c                ó  € TpV P                  V4      pV P                  ! W3/ VB w  rV P                  V4      pW,           pTpV P                  V4      pV P	                  V4      pV P                  V4      pW,           pV# rÇ   )râ   rß   r]   rã   rä   )r2   rº   r’   r”   ÚresidualÚ_s   &&&,  r    rA   ÚViTLayer.forward
  s†   € ð !ˆØ×-Ñ-¨mÓ<ˆØŸ>š>¨-ÑRÈ6ÑRÑˆØŸ™ ]Ó3ˆØ%Õ0ˆð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆØ%Õ0ˆàÐr#   )rß   r]   rã   râ   rä   rÇ   rÈ   rK   s   @@r    rÙ   rÙ     s   ù‡ € ÷>ó >÷÷ ò r#   rÙ   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú	ViTPooleri!  c                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   ÚViTPooler.__annotate__"  s   ø€ ÷ 4ñ 4™yñ 4r#   c                ó¼   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  ,          V n	        R # rÇ   )
r(   r)   r   r°   r0   Úpooler_output_sizeÚdenser   Ú
pooler_actÚ
activationr¶   s   &&€r    r)   ÚViTPooler.__init__"  s>   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3LÑ3LÓMˆŒ
Ü  ×!2Ñ!2Õ3ˆŽr#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÕ   r8   )r   r   s   "€r    r!   rî   '  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r#   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )rl   )rl   é    )rñ   ró   )r2   rº   Úfirst_token_tensorÚpooled_outputs   &&  r    rA   ÚViTPooler.forward'  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr#   )ró   rñ   rÈ   rK   s   @@r    rì   rì   !  s   ù‡ € ÷4ó 4÷
÷ ð r#   rì   c                   ó¢   a a€ ] tR tRt oRtRtRtRtRR.tRt	Rt
RtRtRtR]R]/tR	t]P&                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚViTPreTrainedModeli0  Úvitr6   TrM   rÙ   rº   Ú
attentionsrY   c                ó®  <€ \         SV `  V4       \        V\        4      '       d¯   VP                  e8   \
        P                  ! VP                  RV P                  P                  R7       \
        P                  ! VP                  RV P                  P                  R7       VP                  e#   \
        P                  ! VP                  4       R# R# R# )zInitialize the weightsNr½   )ÚmeanÚstd)r(   Ú_init_weightsr,   rM   rZ   ÚinitÚtrunc_normal_r   Úinitializer_rangerV   rX   Úzeros_)r2   rŽ   r3   s   &&€r    r  Ú ViTPreTrainedModel._init_weightsC  s—   ø€ ô 	‰Ñ˜fÔ%Ü�fœm×,Ò,Ø×)Ñ)Ò5Ü×"Ò" 6×#=Ñ#=ÀCÈTÏ[É[×MjÑMjÕkÜ×Ò˜v×/Ñ/°c¸t¿{¹{×?\Ñ?\Õ]Ø× Ñ Ò,Ü—’˜F×-Ñ-Ö.ñ -ñ	 -r#   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r   r   )r   r   s   "€r    r!   ÚViTPreTrainedModel.__annotate__0  s   ø‡ ‚ áÑò r#   © )Úimage)rC   rD   rE   rF   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphrÙ   r¥   Ú_can_record_outputsÚ_input_embed_layerr9   Úno_gradr  Ú__annotate_func__rH   rI   rJ   rK   s   @@r    rü   rü   0  s   ù‡ € ð ÐØ$€OØ!ÐØ&*Ð#Ø(¨*Ð5ÐØ€NØÐØÐØ"&ÐØ!Ðà˜Ø�lðÐð ,Ðà
‡]‚]ƒ_ô/ó ð/÷) … r#   rü   c                   ó„   a a€ ] tR tRt oR	V3R lV 3R lllt]]! RR7      ]R
V3R lR ll4       4       4       tRt	Vt
V ;t# )ÚViTModeliO  Fc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   r   Úadd_pooling_layerrO   rP   )r   r   s   "€r    r!   ÚViTModel.__annotate__Q  s#   ø€ ÷ ñ ™yð ¹Tð ÑZ^ñ r#   c                ó¢  <€ \         SV `  V4       Wn        \        WR7      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n
        \
        P                  ! VP                  VP                  R7      V n        V'       d   \        V4      MRV n        V P#                  4        R# u upi )zÓ
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
use_mask_token (`bool`, *optional*, defaults to `False`):
    Whether to use a mask token for masked image modeling.
)rO   rÝ   N)r(   r)   r   rM   r`   r   Ú
ModuleListÚrangeÚnum_hidden_layersrÙ   Úlayersrà   r0   rá   Ú	layernormrì   ÚpoolerÚ	post_init)r2   r   r  rO   ré   r3   s   &&&& €r    r)   ÚViTModel.__init__Q  s’   ø€ ô 	‰Ñ˜Ô ØŒÜ'¨ÔNˆŒÜ—m’m¼uÀV×E]ÑE]Ô?^Ó$_Ñ?^¸!¤X¨fÖ%5Ñ?^Ñ$_Ó`ˆŒÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒß+<”i Ô'À$ˆŒà�‰Öùò	 %`s   ÁC)Útie_last_hidden_statesc                óÂ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[/# ©r   r6   Nr€   r}   r’   r”   r7   )r9   r:   r�   rQ   r   r   r	   )r   r   s   "€r    r!   r  d  sv   ø€ ÷  jñ  já—l‘l TÕ)ð jñ ×)Ñ)¨DÕ0ð jñ #'¨¥+ð	 jñ
 Ÿ™ tÕ+ð jñ Ñ+Õ,ð jñ 
$ñ jr#   c                ó¼  € V P                   P                  P                  P                  P                  pVP                  V8w  d   VP                  V4      pV P                  WVR7      p\        V P                  VVR7      pTpV P                   F  p	V	! W„3/ VB pK  	  V P                  V4      p
V P                  e   V P                  V
4      MRp\        W«R7      # )z³
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
)r€   r}   )r   Úinputs_embedsr’   N)Úlast_hidden_stateÚpooler_output)r`   rY   r1   Úweightr˜   rŸ   r   r   r#  r$  r%  r	   )r2   r6   r€   r}   r’   r”   Úexpected_dtypeÚembedding_outputrº   ÚlayerÚsequence_outputrù   s   &&&&&,      r    rA   ÚViTModel.forwarda  sÍ   € ð  Ÿ™×9Ñ9×DÑD×KÑK×QÑQˆØ×Ñ Ô/Ø'Ÿ?™?¨>Ó:ˆLàŸ?™?ØÐTlð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð
 )ˆØ—[”[ˆEÙ! -ÑJÀ6ÑJŠMñ !ð Ÿ.™.¨Ó7ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä)¸OÔiÐir#   )r   r`   r$  r#  r%  )TF©NNNN)rC   rD   rE   rF   r)   r   r   r   rA   rH   rI   rJ   rK   s   @@r    r  r  O  sF   ù‡ € ÷õ ð   Ù¨EÔ2Ø÷ jñ  jó ó 3ó  ÷ jð  jr#   r  ac  
    ViT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )Úcustom_introc                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚViTForMaskedImageModelingi‡  c                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   Ú&ViTForMaskedImageModeling.__annotate__”  s   ø€ ÷ ñ ™yñ r#   c                óf  <€ \         SV `  V4       \        VR RR7      V n        \        P
                  ! \        P                  ! VP                  VP                  ^,          VP                  ,          ^R7      \        P                  ! VP                  4      4      V n        V P                  4        R# )FT)r  rO   )Úin_channelsÚout_channelsr&   N)r(   r)   r  rý   r   Ú
Sequentialr/   r0   Úencoder_strider.   ÚPixelShuffleÚdecoderr&  r¶   s   &&€r    r)   Ú"ViTForMaskedImageModeling.__init__”  s�   ø€ Ü‰Ñ˜Ô ä˜F°eÈDÔQˆŒä—}’}Ü�IŠIØ"×.Ñ.Ø#×2Ñ2°AÕ5¸×8KÑ8KÕKØôô
 �OŠO˜F×1Ñ1Ó2ó
ˆŒð 	�‰Ör#   c                óÂ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[/# r*  )r9   r:   r�   rQ   r   r   r   )r   r   s   "€r    r!   r:  §  sv   ø€ ÷ R
ñ R
á—l‘l TÕ)ðR
ñ ×)Ñ)¨DÕ0ðR
ñ #'¨¥+ð	R
ñ
 Ÿ™ tÕ+ðR
ñ Ñ+Õ,ðR
ñ 
#ñR
r#   c           	     ó€  € Vej   V P                   P                  V P                   P                  8w  d;   \        RV P                   P                   RV P                   P                   R24      hV P                  ! V3RVRVRV/VB pVP
                  pVR,          pVP                  w  r‰p
\        P                  ! V	R,          4      ;r¼VP                  ^ ^^4      P                  WŠW¼4      pV P                  V4      pRpVEe   V P                   P                  V P                   P                  ,          pVP                  RWÿ4      pVP                  V P                   P                  ^4      P                  V P                   P                  ^4      P                  ^4      P                  4       p\         P"                  P%                  WR	R
7      pVV,          P'                  4       VP'                  4       R,           ,          V P                   P(                  ,          p\+        VVVP,                  VP.                  R7      # )aÄ  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

Examples:
```python
>>> from transformers import AutoImageProcessor, ViTForMaskedImageModeling
>>> import torch
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k")
>>> model = ViTForMaskedImageModeling.from_pretrained("google/vit-base-patch16-224-in21k")

>>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
>>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
>>> # create random boolean mask of shape (batch_size, num_patches)
>>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
>>> list(reconstructed_pixel_values.shape)
[1, 3, 224, 224]
```Nz³When `bool_masked_pos` is provided, `patch_size` must be equal to `encoder_stride` to ensure that the reconstructed image has the same dimensions as the input. Got `patch_size` = z and `encoder_stride` = r<   r€   r}   r’   re   Únone)Ú	reductiongñhãˆµøä>)ÚlossÚreconstructionrº   rþ   rm   rn   )r   r+   r?  r>   rý   r-  r=   ÚmathÚfloorrr   rq   rA  r*   Úrepeat_interleaver…   r    r   rs   Úl1_lossÚsumr.   r   rº   rþ   )r2   r6   r€   r}   r’   r”   Úoutputsr3  r‡   Úsequence_lengthr.   ra   rb   Úreconstructed_pixel_valuesÚmasked_im_lossrg   rŠ   Úreconstruction_losss   &&&&&,            r    rA   Ú!ViTForMaskedImageModeling.forward¥  s
  € ðP Ò&¨D¯K©K×,BÑ,BÀdÇkÁk×F`ÑF`Ô,`Üð&à&*§k¡k×&<Ñ&<Ð%=Ð=UÐVZ×VaÑVa×VpÑVpÐUqÐqrðtóð ð /3¯hªhØñ/
à+ð/
ð &>ð/
ð *ð	/
ð
 ñ/
ˆð "×3Ñ3ˆð *¨%Õ0ˆØ4C×4IÑ4IÑ1ˆ
 \ÜŸš O°SÕ$8Ó9Ð9ˆØ)×1Ñ1°!°Q¸Ó:×BÑBÀ:Ð]cÓkˆð &*§\¡\°/Ó%BÐ"àˆØÓ&Ø—;‘;×)Ñ)¨T¯[©[×-CÑ-CÕCˆDØ-×5Ñ5°b¸$ÓEˆOà×1Ñ1°$·+±+×2HÑ2HÈ!ÓLß"Ñ" 4§;¡;×#9Ñ#9¸1Ó=ß‘˜1“ß‘“ð	 ô #%§-¡-×"7Ñ"7¸ÐlrÐ"7Ó"sÐØ1°DÕ8×=Ñ=Ó?À4Ç8Á8Ã:ÐPTÕCTÕUÐX\×XcÑXc×XpÑXpÕpˆNä(ØØ5Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )rA  rý   r5  ©rC   rD   rE   rF   r)   r   r   rA   rH   rI   rJ   rK   s   @@r    r8  r8  ‡  s9   ù‡ € ÷ó ð" Ø÷R
ñ R
ó ó ÷R
ð R
r#   r8  aà  
    ViT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚViTForImageClassificationiü  c                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r    r!   Ú&ViTForImageClassification.__annotate__  s   ø€ ÷ 
ñ 
™yñ 
r#   c                ó8  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP                  ^ 8”  d,   \
        P                  ! VP                  VP                  4      M\
        P                  ! 4       V n	        V P                  4        R# )F)r  N)r(   r)   Ú
num_labelsr  rý   r   r°   r0   ÚIdentityÚ
classifierr&  r¶   s   &&€r    r)   Ú"ViTForImageClassification.__init__  ss   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜F°eÔ<ˆŒð OU×N_ÑN_ÐbcÔNcœ"Ÿ)š) F×$6Ñ$6¸×8IÑ8IÔJÔik×itÒitÓivˆŒð 	�‰Ör#   c                óÂ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[S[,          RS[/# )r   r6   NÚlabelsr}   r’   r”   r7   )r9   r:   rQ   r   r   r
   )r   r   s   "€r    r!   rX    sl   ø€ ÷ #
ñ #
á—l‘l TÕ)ð#
ñ —‘˜tÕ#ð#
ñ #'¨¥+ð	#
ñ
 Ÿ™ tÕ+ð#
ñ Ñ+Õ,ð#
ñ 
ñ#
r#   c                ó
  € V P                   ! V3RVRV/VB pVP                  pVR,          pV P                  V4      p	Rp
Ve   V P                  ! W)V P                  3/ VB p
\        V
V	VP                  VP                  R7      # )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r}   r’   N)rG  Úlogitsrº   rþ   )rl   r÷   rl   )rý   r-  r\  Úloss_functionr   r
   rº   rþ   )r2   r6   r_  r}   r’   r”   rN  r3  rù   ra  rG  s   &&&&&,     r    rA   Ú!ViTForImageClassification.forward  s¡   € ð" /3¯hªhØñ/
à%=ð/
ð *ð/
ð ñ	/
ˆð "×3Ñ3ˆØ'¨Õ0ˆØ—‘ Ó/ˆàˆØÒØ×%Ò% f°d·k±kÑLÀVÑLˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )r\  rZ  rý   r5  rT  rK   s   @@r    rV  rV  ü  s5   ù‡ € ÷
ó 
ð Ø÷#
ñ #
ó ó ÷#
ð #
r#   rV  )Nr½   )rV  r8  r  rü   )5rG   rI  Úcollections.abcr   r   r9   r   Ú r   r  Úactivationsr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr	   r
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_vitr   Ú
get_loggerrC   Úloggerr•   r   rM   r£   r¥   rÊ   rÙ   rì   rü   r  r8  rV  Ú__all__r
  r#   r    Ú<module>rs     sS  ðñ ã ß .ã Ý å &Ý !Ý 6Ý 9÷ñ ÷
 GÝ &ß KÓ Kß IÝ 5Ý (ð 
×	Ò	˜HÓ	%€ôH˜Ÿ™ô Hô<Y�B—I‘Iô Y÷x%ô8.)�2—9‘9ô .)ôbˆR�Y‰Yô ô Ð)ô ô@�—	‘	ô ð ô/˜ó /ó ð/ð< ô4jÐ!ó 4jó ð4jñn ð	ôôf
Ð 2ó f
óðf
ñR ðôô2
Ð 2ó 2
óð2
òj g‚r#   