+
    œ°³jÈ–  ã                   ó   € R t ^ RIHt ^ RIt^ RIHtHt ^RIHt ^RI	H
t
Ht ^RIHt ^RIHtHtHtHtHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHt ^RIHtH t  ^RI!H"t" ^RI#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t*H+t+H,t,H-t- ^RI.H/t/ ]! RR7      ] ! R R]4      4       4       t0 ! R R],4      t1 ! R R])4      t2 ! R R]Pf                  4      t4 ! R R](4      t5 ! R R]+4      t6 ! R  R!]&4      t7 ! R" R#]*4      t8] ! R$ R%]-4      4       t9] ! R& R']94      4       t: ! R( R)]Pf                  4      t;]! R*R7       ! R+ R,]94      4       t<]! R-R7       ! R. R/]94      4       t= ! R0 R1]$4      t> ! R2 R3]Pf                  4      t? ! R4 R5]Pf                  4      t@ ! R6 R7]Pf                  4      tA ! R8 R9]Pf                  4      tB ! R: R;]Pf                  4      tC ! R< R=]Pf                  4      tD] ! R> R?]94      4       tE]! R@R7       ! RA RB]
]94      4       tF. RCOtGR# )DzPyTorch BEiT model.)Ú	dataclassN)ÚTensorÚnn)Úinitialization)ÚBackboneMixinÚfilter_output_hidden_states)Úcreate_bidirectional_mask)ÚBackboneOutputÚBaseModelOutputWithPoolingÚImageClassifierOutputÚMaskedLMOutputÚSemanticSegmenterOutput)ÚPreTrainedModel)ÚUnpack)Ú#compile_compatible_method_lru_cache)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputs)ÚResNetConvLayer)ÚSwinDropPath)ÚViTAttentionÚViTEmbeddingsÚViTLayerÚViTMLPÚViTPatchEmbeddingsÚViTPreTrainedModel©Ú
BeitConfigz-
    Class for outputs of [`BeitModel`].
    )Úcustom_introc                   ó   € ] tR t^+tRtRtR# )ÚBeitModelOutputWithPoolinga2  
pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
    Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
    *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
    will be returned.
© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__static_attributes__r$   ó    Úf/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/beit/modular_beit.pyr#   r#   +   s   † õr+   r#   c                   ó   € ] tR t^:tRtR# )ÚBeitPatchEmbeddingsr$   N©r%   r&   r'   r(   r*   r$   r+   r,   r.   r.   :   ó   † Ûr+   r.   c                   óH   a € ] tR t^>t o V 3R lR ltRV 3R lR lltRtV tR# )ÚBeitEmbeddingsc                ó$   <€ V ^8„  d   QhRS[ RR/# ©é   ÚconfigÚreturnNr   )ÚformatÚ__classdict__s   "€r,   Ú__annotate__ÚBeitEmbeddings.__annotate__?   s   ø€ ÷ >ñ >™zð >¨dñ >r+   c                ó¼  € \         P                  P                  V 4       \         P                  ! \        P
                  ! ^^VP                  4      4      V n        VP                  '       d7   \         P                  ! \        P
                  ! ^^VP                  4      4      MRV n	        \        V4      V n        VP                  V n        V P                  P                  pVP                  '       d>   \         P                  ! \        P
                  ! ^V^,           VP                  4      4      MRV n        \         P                   ! VP"                  4      V n        R# )é   N)r   ÚModuleÚ__init__Ú	ParameterÚtorchÚzerosÚhidden_sizeÚ	cls_tokenÚuse_mask_tokenÚ
mask_tokenr.   Úpatch_embeddingsÚ
patch_sizeÚnum_patchesÚ use_absolute_position_embeddingsÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropout)Úselfr6   rI   s   && r,   r?   ÚBeitEmbeddings.__init__?   sé   € Ü
�	‰	×Ñ˜4Ô äŸš¤e§k¢k°!°Q¸×8JÑ8JÓ&KÓLˆŒØQW×Qf×QfÐQfœ"Ÿ,š,¤u§{¢{°1°a¸×9KÑ9KÓ'LÔMÐlpˆŒÜ 3°FÓ ;ˆÔØ ×+Ñ+ˆŒØ×+Ñ+×7Ñ7ˆð ×6×6Ð6ô �LŠLœŸš Q¨°a­¸×9KÑ9KÓLÔMàð 	Ô ô
 —z’z &×"<Ñ"<Ó=ˆŽr+   Nc                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )r5   Úpixel_valuesÚbool_masked_posNr7   )rA   r   Ú
BoolTensor)r8   r9   s   "€r,   r:   r;   N   s<   ø€ ÷ ñ á—l‘lðñ ×)Ñ)¨DÕ0ðñ 
�‰ñ	r+   c                ó  € VP                   w   r4pV P                  V4      pVP                  4       w  rxpVeZ   V P                  P	                  WxR4      p	VP                  R4      P                  V	4      p
V^V
,
          ,          Wš,          ,           pV P                  P	                  VRR4      p\        P                  ! W¶3^R7      pV P                  e   W`P                  WdV4      ,           pV P                  V4      pV# )N©Údiméÿÿÿÿ)ÚshaperG   ÚsizerF   ÚexpandÚ	unsqueezeÚtype_asrD   rA   ÚcatrK   Úinterpolate_pos_encodingrN   )rO   rR   rS   Ú_ÚheightÚwidthÚ
embeddingsÚ
batch_sizeÚseq_lenÚmask_tokensÚmaskÚ
cls_tokenss   &&&         r,   ÚforwardÚBeitEmbeddings.forwardN   så   € ð
 +×0Ñ0Ñˆˆ1�eØ×*Ñ*¨<Ó8ˆ
Ø!+§¡Ó!2Ñˆ
˜QàÒ&ØŸ/™/×0Ñ0°ÀbÓIˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# q¨4¥xÕ0°;Õ3EÕEˆJà—^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ü—Y’Y 
Ð7¸QÔ?ˆ
à×#Ñ#Ò/Ø#×&CÑ&CÀJÐX]Ó&^Õ^ˆJà—\‘\ *Ó-ˆ
àÐr+   )rD   rN   rF   rG   rH   rK   ©N)r%   r&   r'   r(   r?   ri   r*   Ú__classdictcell__©r9   s   @r,   r2   r2   >   s   ø‡ € ÷>ð >÷÷ ð r+   r2   c                   óˆ   a a€ ] tR t^ht oV3R lV 3R llt]]! ^
R7      V3R lR l4       4       tR	V3R lR lltRt	Vt
V ;t# )
ÚBeitRelativePositionBiasc                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   Ú%BeitRelativePositionBias.__annotate__i   s   ø€ ÷ 	
ñ 	
™zð 	
¨dñ 	
r+   c                ó  <€ \         SV `  4        VP                  p\        V\        \
        34      '       g   W"3pV^ ,          VP                  ,          V^,          VP                  ,          3V n        ^V P                  ^ ,          ,          ^,
          ^V P                  ^,          ,          ^,
          ,          ^,           V n        \        P                  ! \        P                  ! V P                  VP                  4      4      V n        R# )é    N)Úsuperr?   Ú
image_sizeÚ
isinstanceÚtupleÚlistrH   Úwindow_sizeÚnum_relative_distancer   r@   rA   rB   Únum_attention_headsÚrelative_position_bias_table)rO   r6   ru   Ú	__class__s   && €r,   r?   Ú!BeitRelativePositionBias.__init__i   sÈ   ø€ Ü‰ÑÔØ×&Ñ&ˆ
Ü˜*¤u¬d m×4Ò4Ø$Ð1ˆJØ& q�M¨V×->Ñ->Õ>À
È1ÅÐQW×QbÑQbÕ@bÐcˆÔØ&'¨$×*:Ñ*:¸1Õ*=Õ&=ÀÕ&AÀaÈ$×JZÑJZÐ[\ÕJ]ÕF]Ð`aÕFaÕ%bÐefÕ%fˆÔ"Ü,.¯LªLÜ�KŠK˜×2Ñ2°F×4NÑ4NÓOó-
ˆÖ)r+   )Úmaxsizec                óP   <€ V ^8„  d   QhRS[ S[S[3,          RS[P                  /# )r5   ry   r7   )rw   ÚintrA   r   )r8   r9   s   "€r,   r:   rq   w   s'   ø€ ÷ 'ñ '±e¹CÁ¸Hµoð 'É%Ï,É,ñ 'r+   c                ó¸  € ^V ^ ,          ,          ^,
          ^V ^,          ,          ^,
          ,          ^,           pV ^ ,          V ^,          ,          p\         P                  ! \         P                  ! \         P                  ! \         P                  ! V ^ ,          4      \         P                  ! V ^,          4      RR7      4      ^R7      pVR,          VR,          ,
          P                  ^^^ 4      P                  4       pVR;;,          V ^ ,          ^,
          ,          uu&   VR;;,          V ^,          ^,
          ,          uu&   VR;;,          ^V ^,          ,          ^,
          ,          uu&   \         P                  ! V^,           3^,          VP                  R7      pVP                  R	4      VR
&   V^,
          VR&   V^,
          VR&   V^,
          VR&   V# )z§
This method creates the relative position index, modified to support arbitrary window sizes,
as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
Úij)Úindexing)Ú	start_dim)rZ   Údtype)ºNNNr‡   N)r‡   Nr‡   )r‡   r‡   rs   )r‡   r‡   r=   rX   )ºr=   NNrˆ   )rs   ºrs   NN)r‰   rs   )rs   rs   )
rA   ÚflattenÚstackÚmeshgridÚarangeÚpermuteÚ
contiguousrB   r†   Úsum)ry   rz   Úwindow_areaÚcoords_flattenÚrelative_coordsÚrelative_position_indexs   &     r,   Ú generate_relative_position_indexÚ9BeitRelativePositionBias.generate_relative_position_indexu   s  € ð "# [°¥^Õ!3°aÕ!7¸AÀÈAÅÕ<NÐQRÕ<RÕ SÐVWÕ WÐØ! !•n {°1¥~Õ5ˆô ŸšÜ�KŠKœŸš¤u§|¢|°KÀµNÓ'CÄUÇ\Â\ÐR]Ð^_ÕR`ÓEaÐlpÔqÓrØô
ˆð *¨*Õ5¸ÀzÕ8RÕR×[Ñ[Ð\]Ð_`ÐbcÓd×oÑoÓqˆà˜×  K°¥N°QÕ$6Õ6Ó Ø˜×  K°¥N°QÕ$6Õ6Ó Ø˜×  A¨°A­Õ$6¸Õ$:Õ:Ó ä"'§+¢+°KÀ!µOÐ3EÈÕ3IÐQ`×QfÑQfÔ"gÐØ*9×*=Ñ*=¸bÓ*AÐ Ñ'Ø)>ÀÕ)BÐ Ñ&Ø)>ÀÕ)BÐ Ñ&Ø(=ÀÕ(AÐ Ñ%Ø&Ð&r+   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# )r5   r_   r7   )ÚboolrA   r   )r8   r9   s   "€r,   r:   rq   ‘   s"   ø€ ÷ -3ñ -3¹Tð -3Ñ]b×]iÑ]iñ -3r+   c                ót  € ^V P                   ^ ,          ,          ^,
          p^V P                   ^,          ,          ^,
          p^V^ ,          ,          ^,
          p^V^,          ,          ^,
          pV P                  pV P                  p	Wg,          ^,           p
VRV	^,
           pVP                  ^WTR4      P	                  ^ ^^^4      p\
        P                  P                  V\        V4      \        V4      3RR7      pVP	                  ^ ^^^4      P                  V
^,
          R4      p\        P                  ! WÈV	^,
          R .4      pV P                  V4      pWÞP                  R4      ,          pVP                  V^ ,          V^,          ,          ^,           V^ ,          V^,          ,          ^,           R4      pVP	                  ^^ ^4      P                  4       pV'       dC   \
        P                  P                  VP                  ^4      W33RRR7      P                  ^4      pVP                  ^ 4      # )ze
Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
NÚbilinear)rZ   ÚmodeF©rZ   r›   Úalign_cornersrX   )ry   r|   rz   ÚreshaperŽ   r   Ú
functionalÚinterpolater   rA   r^   r•   Úviewr�   r\   Úsqueeze)rO   ry   r_   Údim_sizeÚ
old_heightÚ	old_widthÚ
new_heightÚ	new_widthÚ old_relative_position_bias_tableÚold_num_relative_distanceÚnew_num_relative_distanceÚold_sub_tableÚnew_sub_tableÚ new_relative_position_bias_tabler”   Úrelative_position_biass   &&&&            r,   ri   Ú BeitRelativePositionBias.forward‘   s!  € ð ˜×)Ñ)¨!Õ,Õ,¨qÕ0ˆ
Ø˜×(Ñ(¨Õ+Õ+¨aÕ/ˆ	à˜ Q�Õ'¨!Õ+ˆ
Ø˜ A�Õ&¨Õ*ˆ	à+/×+LÑ+LÐ(à$(×$>Ñ$>Ð!Ø$.Õ$:¸QÕ$>Ð!à8Ð9XÐ;TÐWXÕ;XÐYˆà%×-Ñ-¨a°ÈÓK×SÑSÐTUÐWXÐZ[Ð]^Ó_ˆÜŸ™×1Ñ1Ø¤¨:Ó!6¼	À)Ó8LÐ MÐT^ð 2ó 
ˆð &×-Ñ-¨a°°A°qÓ9×AÑAÐB[Ð^_ÕB_ÐacÓdˆä+0¯9ª9ØÐ=VÐYZÕ=ZÐ=\Ð]Ð^ó,
Ð(ð #'×"GÑ"GÈÓ"TÐØ!A×B^ÑB^Ð_aÓBbÕ!cÐð "8×!<Ñ!<Ø˜�N˜[¨�^Õ+¨aÕ/°¸QµÀ+ÈaÅ.Õ1PÐSTÕ1TÐVXó"
Ðð "8×!?Ñ!?ÀÀ1ÀaÓ!H×!SÑ!SÓ!UÐç#Ü%'§]¡]×%>Ñ%>Ø&×0Ñ0°Ó3ØÐ)ØØ#ð	 &?ó &÷
 ‰g�a‹jð #ð &×/Ñ/°Ó2Ð2r+   )rz   r|   ry   )FN)r%   r&   r'   r(   r?   Ústaticmethodr   r•   ri   r*   rl   Ú__classcell__©r}   r9   s   @@r,   ro   ro   h   s?   ù‡ € ÷	
ó 	
ð Ù(°Ô4÷'ó 5ó ð'÷4-3÷ -3ò -3r+   ro   c                   ó>   a a€ ] tR t^Át oV3R lV 3R lltRtVtV ;t# )ÚBeitAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r5   r6   r   )r8   r9   s   "€r,   r:   ÚBeitAttention.__annotate__Â   s   ø€ ÷ `ñ `™zñ `r+   c                ó2  <€ \         SV `  V4       \        P                  ! VP                  VP
                  V P                  ,          4      V n        \        P                  ! VP                  VP
                  V P                  ,          R R7      V n        \        P                  ! VP                  VP
                  V P                  ,          4      V n	        \        P                  ! VP
                  V P                  ,          VP                  4      V n
        R# )F)ÚbiasN)rt   r?   r   ÚLinearrC   r{   Úhead_dimÚq_projÚk_projÚv_projÚo_proj©rO   r6   r}   s   &&€r,   r?   ÚBeitAttention.__init__Â   s¸   ø€ Ü‰Ñ˜Ô Ü—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ó_ˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^ÐejÔkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Õ4^Ó_ˆŒÜ—i’i × :Ñ :¸T¿]¹]Õ JÈF×L^ÑL^Ó_ˆŽr+   )r½   r¿   r¼   r¾   ©r%   r&   r'   r(   r?   r*   rl   r±   r²   s   @@r,   r´   r´   Á   s   ù‡ € ÷`÷ `ó `r+   r´   c                   ó   € ] tR t^ÊtRtR# )ÚBeitMLPr$   Nr/   r$   r+   r,   rÄ   rÄ   Ê   r0   r+   rÄ   c                   ó   € ] tR t^ÎtRtR# )ÚBeitDropPathr$   Nr/   r$   r+   r,   rÆ   rÆ   Î   r0   r+   rÆ   c                   ó\   a a€ ] tR t^Òt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	Ú	BeitLayerz?This corresponds to the Block class in the timm implementation.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r5   r6   Údrop_path_rate)r    Úfloat)r8   r9   s   "€r,   r:   ÚBeitLayer.__annotate__Õ   s   ø€ ÷ vñ v™zð v¹5ñ vr+   c                ó8  <€ \         SV `  4        VP                  V n        VR 8”  d   \        V4      M\        P
                  ! 4       V n        VP                  pV^ 8”  d>   \        P                  ! V\        P                  ! VP                  4      ,          RR7      MRV n        V^ 8”  d>   \        P                  ! V\        P                  ! VP                  4      ,          RR7      MRV n        VP                  '       d   \        V4      V n        R# RV n        R# )ç        T)Úrequires_gradg      ð?N)rt   r?   rH   rÆ   r   ÚIdentityÚ	drop_pathÚlayer_scale_init_valuer@   rA   ÚonesrC   Úlambda_1Úlambda_2Úuse_relative_position_biasro   r®   )rO   r6   rÊ   Úinit_valuesr}   s   &&& €r,   r?   ÚBeitLayer.__init__Õ   sØ   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ9GÈ#Ô9Mœ nÔ5ÔSU×S^ÒS^ÓS`ˆŒà×3Ñ3ˆà^iÐlmÔ^mŒB�LŠL˜¤u§z¢z°&×2DÑ2DÓ'EÕEÐUYÕZÐsvð 	Œð _jÐlmÔ^mŒB�LŠL˜¤u§z¢z°&×2DÑ2DÓ'EÕEÐUYÕZÐsvð 	Œð KQ×Jk×JkÐJkÔ&>¸vÓ&FˆÖ#ÐquˆÖ#r+   c                ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[S[S[3,          R,          RS[S[,          RS[ P                  /# )r5   Úhidden_statesÚattention_maskNr_   Ú
resolutionÚkwargsr7   )rA   r   r˜   rw   r�   r   r   )r8   r9   s   "€r,   r:   rÌ   ã   sk   ø€ ÷ &ñ &á—|‘|ð&ñ Ÿ™ tÕ+ð&ñ #'ð	&ñ
 ™#™s˜(•O dÕ*ð&ñ Ñ+Õ,ð&ñ 
�‰ñ&r+   c                ób  € V P                   e[   Vw  rgW`P                  ,          WpP                  ,          3pV P                  WƒVP                  ^,          R7      p	Ve	   W’,           MT	pTp
V P                  V4      pV P                  ! V3RV/VB w  rV P                  V4      pV P                  V,          pV P                  V4      V
,           pTp
V P                  V4      pV P                  V4      pV P                  V4      pV P                  V,          pV P                  V4      V
,           pV# )N)r£   rÛ   )r®   rH   rY   Úlayernorm_beforeÚ	attentionrN   rÔ   rÑ   Úlayernorm_afterÚmlprÕ   )rO   rÚ   rÛ   r_   rÜ   rÝ   ra   rb   ry   r®   Úresidualr`   s   &&&&&,      r,   ri   ÚBeitLayer.forwardã   s6  € ð ×&Ñ&Ò2Ø&‰MˆFØ!§_¡_Õ4°e¿¹Õ6NÐOˆKØ%)×%@Ñ%@ØÀ×@SÑ@SÐTUÕ@Vð &Aó &Ð"ð <JÒ;UÐ&Ö7Ð[qð ð
 !ˆØ×-Ñ-¨mÓ<ˆØŸ>š>Øñ
à)ð
ð ñ
Ñˆð
 Ÿ™ ]Ó3ˆØŸ™¨Õ5ˆØŸ™ }Ó5¸Õ@ˆð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆØŸ™¨Õ5ˆØŸ™ }Ó5¸Õ@ˆàÐr+   )rÑ   rÔ   rÕ   rH   r®   )rÎ   ©NFN©
r%   r&   r'   r(   r)   r?   ri   r*   rl   r±   r²   s   @@r,   rÈ   rÈ   Ò   s#   ù‡ € ÙI÷võ v÷&÷ &ò &r+   rÈ   c                   ó:   a € ] tR tRt o R.tR.tRtRtR tRt	V t
R# )ÚBeitPreTrainedModeli  rÈ   z.*relative_position_index.*Fc                ó*  € \         P                  ! W4       \        V\        4      '       d�   \        P
                  ! VP                  4       VP                  e!   \        P
                  ! VP                  4       VP                  e#   \        P
                  ! VP                  4       R# R# \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d™   \        VP                  \        P                  4      '       dm   \        P                  ! VP                  V P                   P"                  4       \        P                  ! VP$                  V P                   P"                  4       R# R# R# )zInitialize the weightsN)r   Ú_init_weightsrv   r2   ÚinitÚzeros_rD   rF   rK   ro   r|   rÈ   rÔ   r   r@   Ú	constant_r6   rÒ   rÕ   )rO   Úmodules   &&r,   rê   Ú!BeitPreTrainedModel._init_weights  sô   € ä×%Ò% dÔ3Ü�fœn×-Ò-Ü�KŠK˜×(Ñ(Ô)Ø× Ñ Ò,Ü—’˜F×-Ñ-Ô.Ø×)Ñ)Ò5Ü—’˜F×6Ñ6Ö7ñ 6ä˜Ô 8×9Ò9Ü�KŠK˜×;Ñ;Ö<Ü˜¤	×*Ò*Ü˜&Ÿ/™/¬2¯<©<×8Ò8Ü—’˜vŸ™°·±×0RÑ0RÔSÜ—’˜vŸ™°·±×0RÑ0RÖSñ 9ñ +r+   r$   N)r%   r&   r'   r(   Ú_no_split_modulesÚ"_keys_to_ignore_on_load_unexpectedÚ_supports_flash_attnÚ_supports_flex_attnrê   r*   rl   rm   s   @r,   rè   rè     s.   ø‡ € à$˜ÐØ*HÐ)IÐ&Ø ÐØÐ÷Tð Tr+   rè   c                   ó„   a a€ ] tR tRt oR	V3R lV 3R lllt]]! RR7      ]R
V3R lR ll4       4       4       tRt	Vt
V ;t# )Ú	BeitModeli$  c                ó*   <€ V ^8„  d   QhRS[ RS[RR/# )r5   r6   Úadd_pooling_layerr7   N)r    r˜   )r8   r9   s   "€r,   r:   ÚBeitModel.__annotate__&  s"   ø€ ÷ ñ ™zð ¹dð Èdñ r+   c           
     óÈ  <€ \         SV `  V4       Wn        \        V4      V n        VP
                  '       d   \        V4      MRV n        \        VP                  4       Uu. uF7  q1P                  V,          \        VP                  ^,
          ^4      ,          NK9  	  pp\        P                  ! V Uu. uF  p\        WR7      NK  	  up4      V n        VP                   '       d   \        P"                  ! 4       M+\        P$                  ! VP&                  VP(                  R7      V n        V'       d   \-        V4      MRV n        V P1                  4        R# u upi u upi )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
N)rÊ   ©Úeps)rt   r?   r6   r2   rc   Ú!use_shared_relative_position_biasro   Úshared_position_biasÚrangeÚnum_hidden_layersrÊ   Úmaxr   Ú
ModuleListrÈ   ÚlayersÚuse_mean_poolingrÐ   Ú	LayerNormrC   Úlayer_norm_epsÚ	layernormÚ
BeitPoolerÚpoolerÚ	post_init)rO   r6   r÷   ÚiÚdrop_path_ratesÚrr}   s   &&&   €r,   r?   ÚBeitModel.__init__&  s  ø€ ô
 	‰Ñ˜Ô ØŒä(¨Ó0ˆŒà06×0X×0XÐ0XÔ$ VÔ,Ð^bð 	Ô!ô W\Ð\b×\tÑ\tÔVuó
ÙVuÐQR×!Ñ! AÕ%¬¨F×,DÑ,DÀqÕ,HÈ!Ó(L×LÐLÑVuð 	ð 
ô —m’mÑRaÓ$bÑRaÈQ¤Y¨v×%HÑRaÑ$bÓcˆŒð $×4×4Ð4ŒB�KŠKŒM¼"¿,º,Àv×GYÑGYÐ_e×_tÑ_tÔ:uð 	Œ÷ ->”j Ô(À4ˆŒð 	�‰Öùò
ùò %cs   Á"=EÂ5EF)Útie_last_hidden_statesc                ó¦   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[ P                  R,          RS[S[,          RS[/# )r5   rR   rS   Nr_   rÛ   rÝ   r7   )rA   r   rT   r˜   r   r   r#   )r8   r9   s   "€r,   r:   rø   B  sm   ø€ ÷ -jñ -já—l‘lð-jñ ×)Ñ)¨DÕ0ð-jñ #'ð	-jñ
 Ÿ™ tÕ+ð-jñ Ñ+Õ,ð-jñ 
$ñ-jr+   c           
     ó8  € V P                  WR7      pVP                  R,          p\        V P                  VVR7      pV P                  eo   Vw  r‰W€P                  P
                  ,          W�P                  P
                  ,          3p
V P	                  W£VP                  ^,          R7      pVe	   W´,           MTpTpV P                   F  pV! V3RVRVRV/VB pK  	  V P                  V4      pV P                  e   V P                  V4      MRp\        WïR	7      # )
z³
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
)rS   ºr5   NN)r6   Úinputs_embedsrÛ   N)r_   r£   rÛ   r_   rÜ   )Úlast_hidden_stateÚpooler_output)
rc   rY   r   r6   rý   rH   r  r  r  r#   )rO   rR   rS   r_   rÛ   rÝ   Úembedding_outputrÜ   ra   rb   ry   Úshared_relative_position_biasrÚ   ÚlayerÚsequence_outputÚpooled_outputs   &&&&&,          r,   ri   ÚBeitModel.forward?  s4  € ð  Ÿ?™?¨<˜?ÓYÐØ!×'Ñ'¨Õ+ˆ
ä2Ø—;‘;Ø*Ø)ô
ˆð ×$Ñ$Ò0Ø&‰MˆFØ!§[¡[×%;Ñ%;Õ;¸UÇkÁk×F\ÑF\Õ=\Ð]ˆKØ,0×,EÑ,EØÐYi×YoÑYoÐpqÕYrð -Fó -Ð)ð
 "Ò-ð .Ö>à2ð ð )ˆØ—[”[ˆEÙ!Øñà-ðð *Bðð &ð	ð
 ñŠMñ !ð Ÿ.™.¨Ó7ˆØ8<¿¹Ò8O˜Ÿ™ OÔ4ÐUYˆä)¸OÔiÐir+   )r6   rc   r  r  r  rý   )Trå   )r%   r&   r'   r(   r?   r   r   r   ri   r*   rl   r±   r²   s   @@r,   rõ   rõ   $  sF   ù‡ € ÷õ ð2  Ù¨EÔ2Ø÷-jñ -jó ó 3ó  ÷-jð -jr+   rõ   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )r  ir  c                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   ÚBeitPooler.__annotate__s  s   ø€ ÷ 
ñ 
™zð 
¨dñ 
r+   c                ó¼   <€ \         SV `  4        VP                  '       d4   \        P                  ! VP
                  VP                  R 7      V n        R# RV n        R# )rú   N)rt   r?   r  r   r  rC   r  r  rÀ   s   &&€r,   r?   ÚBeitPooler.__init__s  sD   ø€ Ü‰ÑÔàKQ×Kb×KbÐKbŒB�LŠL˜×+Ñ+°×1FÑ1FÔGð 	ŽØhlð 	Žr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r5   rÚ   r7   ©rA   r   )r8   r9   s   "€r,   r:   r  y  s'   ø€ ÷ vñ v¡U§\¡\ð v±e·l±lñ vr+   c                ó~   € V P                   e(   V P                  VR,          P                  ^4      4      # VR,          # )N©r‡   rˆ   r‡   )r‡   rs   )r  Úmean©rO   rÚ   s   &&r,   ri   ÚBeitPooler.forwardy  s6   € àBFÇ.Á.ÒB\ˆt�~‰~˜m¨HÕ5×:Ñ:¸1Ó=Ó>ÐuÐboÐptÕbuÐur+   )r  ©	r%   r&   r'   r(   r?   ri   r*   rl   r±   r²   s   @@r,   r  r  r  s!   ù‡ € ÷
ó 
÷v÷ vð vr+   r  aÔ  
    Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting
    visual tokens of a Vector-Quantize Variational Autoencoder (VQ-VAE), whereas other vision models like ViT and DeiT
    predict RGB pixel values. As a result, this class is incompatible with [`AutoModelForMaskedImageModeling`], so you
    will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT.
    c                   ón   a a€ ] tR tRt oV3R lV 3R lltR t]]RV3R lR ll4       4       tRt	Vt
V ;t# )	ÚBeitForMaskedImageModelingi~  c                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   Ú'BeitForMaskedImageModeling.__annotate__‡  s   ø€ ÷ ñ ™zð ¨dñ r+   c                óN  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \
        P                  ! VP                  VP                  R7      V n	        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# )F©r÷   rú   N)rt   r?   Ú
num_labelsrõ   Úbeitr   r  rC   r  r  rº   Ú
vocab_sizeÚlm_headr	  rÀ   s   &&€r,   r?   Ú#BeitForMaskedImageModeling.__init__‡  su   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô>ˆŒ	ô Ÿš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÓGˆŒð 	�‰Ör+   c                ó   € R # rk   r$   )rO   s   &r,   Úget_output_embeddingsÚ0BeitForMaskedImageModeling.get_output_embeddings”  s   € Ùr+   c                óì   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[RS[ P                  R,          RS[S[,          RS[S[,          /# )	r5   rR   NrS   Úlabelsr_   rÛ   rÝ   r7   )rA   r   rT   r˜   r   r   rw   r   )r8   r9   s   "€r,   r:   r,  ™  s‹   ø€ ÷ @
ñ @
á—l‘l TÕ)ð@
ñ ×)Ñ)¨DÕ0ð@
ñ —‘˜tÕ#ð	@
ñ
 #'ð@
ñ Ÿ™ tÕ+ð@
ñ Ñ+Õ,ð@
ñ 
‘Õ	ñ@
r+   c           	     ó8  € V P                   ! V3RVRVRV/VB pVP                  pV P                  V4      pV P                  VR,          4      p	Rp
Ve%   \        P
                  ! 4       pV! W’,          V4      p
\        V
V	VP                  VP                  R7      # )a½  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
>>> import torch
>>> from PIL import Image
>>> import requests

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
>>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

>>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
>>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
>>> # create random boolean mask of shape (batch_size, num_patches)
>>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss, logits = outputs.loss, outputs.logits
>>> list(logits.shape)
[1, 196, 8192]
```rS   r_   rÛ   N©ÚlossÚlogitsrÚ   Ú
attentions©r‡   rˆ   )	r0  r  r  r2  r   ÚCrossEntropyLossr   rÚ   r=  )rO   rR   rS   r8  r_   rÛ   rÝ   Úoutputsr  Úprediction_scoresÚmasked_lm_lossÚloss_fcts   &&&&&&,     r,   ri   Ú"BeitForMaskedImageModeling.forward—  s¹   € ðX —)’)Øñ
à+ð
ð &>ð
ð *ð	
ð
 ñ
ˆð "×3Ñ3ˆØŸ.™.¨Ó9ˆØ ŸL™L¨¸Õ)?Ó@ÐàˆØÒÜ×*Ò*Ó,ˆHÙ%Ð&7Õ&HÈ&ÓQˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )r0  r  r2  r/  )NNNFN)r%   r&   r'   r(   r?   r5  r   r   ri   r*   rl   r±   r²   s   @@r,   r*  r*  ~  s>   ù‡ € ÷ó òð Ø÷@
ñ @
ó ó ÷@
ð @
r+   r*  z¶
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                   óh   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tRtVt	V ;t
# )ÚBeitForImageClassificationiÜ  c                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   Ú'BeitForImageClassification.__annotate__ã  s   ø€ ÷ 
ñ 
™zð 
¨dñ 
r+   c                ó8  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        VP                  ^ 8”  d,   \
        P                  ! VP                  VP                  4      M\
        P                  ! 4       V n	        V P                  4        R# )Tr.  N)rt   r?   r/  rõ   r0  r   rº   rC   rÐ   Ú
classifierr	  rÀ   s   &&€r,   r?   Ú#BeitForImageClassification.__init__ã  ss   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô=ˆŒ	ð OU×N_ÑN_ÐbcÔNcœ"Ÿ)š) F×$6Ñ$6¸×8IÑ8IÔJÔik×itÒitÓivˆŒð 	�‰Ör+   c                óœ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[RS[S[,          RS[S[,          /# ©r5   rR   Nr8  r_   rÝ   r7   )rA   r   r˜   r   r   rw   r   )r8   r9   s   "€r,   r:   rH  ñ  s[   ø€ ÷  
ñ  
á—l‘l TÕ)ð 
ñ —‘˜tÕ#ð 
ñ #'ð	 
ñ
 Ñ+Õ,ð 
ñ 
Ñ&Õ	&ñ 
r+   c                óð   € V P                   ! V3RV/VB pVP                  pV P                  V4      pRpVe   V P                  W'V P                  4      p\        VVVP                  VP                  R7      # )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
r_   Nr:  )r0  r  rJ  Úloss_functionr6   r   rÚ   r=  )	rO   rR   r8  r_   rÝ   r@  r  r<  r;  s	   &&&&,    r,   ri   Ú"BeitForImageClassification.forwardï  sˆ   € ð —)’)Øñ
à%=ð
ð ñ
ˆð  ×-Ñ-ˆà—‘ Ó/ˆàˆØÒØ×%Ñ% f°d·k±kÓBˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )r0  rJ  r/  ©NNF)r%   r&   r'   r(   r?   r   r   ri   r*   rl   r±   r²   s   @@r,   rF  rF  Ü  s5   ù‡ € ÷
ó 
ð Ø÷ 
ñ  
ó ó ÷ 
ð  
r+   rF  c                   óB   a a€ ] tR tRt oRV3R lV 3R llltRtVtV ;t# )ÚBeitConvLayeri  c                óÒ   <€ V ^8„  d   QhRS[ RS[ RS[ S[S[ S[ 3,          ,          RS[ RS[ S[S[ S[ 3,          ,          S[,          RS[RS[ S[S[ S[ 3,          ,          RS[ R	S[/	# )
r5   Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr¹   ÚdilationÚgroupsÚ
activation)r�   rw   Ústrr˜   )r8   r9   s   "€r,   r:   ÚBeitConvLayer.__annotate__  s‘   ø€ ÷ 
ñ 
áð
ñ ð
ñ ™5¡¡c �?Õ*ð	
ñ
 ð
ñ ‘u™S¡#˜X•Õ&©Õ,ð
ñ ð
ñ ™™c¡3˜h�Õ'ð
ñ ð
ñ ñ
r+   c
                ój   <€ \         S
V `  4        \        P                  ! VVVVVVVVR 7      V n        R# ))rU  rV  rW  rX  rY  rZ  r[  r¹   N)rt   r?   r   ÚConv2dÚconvolution)rO   rU  rV  rW  rX  rY  r¹   rZ  r[  r\  r}   s   &&&&&&&&&&€r,   r?   ÚBeitConvLayer.__init__  s9   ø€ ô 	‰ÑÔÜŸ9š9Ø#Ø%Ø#ØØØØØô	
ˆÖr+   )ra  )é   r=   rs   Fr=   r=   ÚrelurÂ   r²   s   @@r,   rS  rS    s   ù‡ € ÷
÷ 
õ 
r+   rS  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚBeitPyramidPoolingBlocki.  c                ó0   <€ V ^8„  d   QhRS[ RS[ RS[ RR/# )r5   Ú
pool_scalerU  Úchannelsr7   N©r�   )r8   r9   s   "€r,   r:   Ú$BeitPyramidPoolingBlock.__annotate__/  s/   ø€ ÷ Hñ H¡3ð H±Sð HÁCð HÈDñ Hr+   c                ó~   <€ \         SV `  4        \        P                  ! V4      V n        \        W#^R7      V n        R# )r=   ©rW  N)rt   r?   r   ÚAdaptiveAvgPool2dÚpoolingrS  Úconv)rO   rh  rU  ri  r}   s   &&&&€r,   r?   Ú BeitPyramidPoolingBlock.__init__/  s.   ø€ Ü‰ÑÔÜ×+Ò+¨JÓ7ˆŒÜ! +ÀQÔGˆŽ	r+   c                ój   <€ V ^8„  d   QhRS[ P                  RS[S[S[3,          RS[ P                  /# )r5   ÚinputrZ   r7   )rA   r   rw   r�   )r8   r9   s   "€r,   r:   rk  4  s2   ø€ ÷ ñ ™UŸ\™\ð ±±s¹C°xµð ÁUÇ\Á\ñ r+   c                óŽ   € V P                  V4      pV P                  V4      p\        P                  P	                  W2R RR7      pV# )rš   Frœ   )ro  rp  r   rŸ   r    )rO   rs  rZ   Úhidden_states   &&& r,   ri   ÚBeitPyramidPoolingBlock.forward4  s@   € Ø—|‘| EÓ*ˆØ—y‘y Ó.ˆÜ—}‘}×0Ñ0°ÈzÐinÐ0ÓoˆØÐr+   )rp  ro  r(  r²   s   @@r,   rf  rf  .  s    ù‡ € ÷Hó H÷
÷ ð r+   rf  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚBeitPyramidPoolingModulei;  aK  
Pyramid Pooling Module (PPM) used in PSPNet.

Args:
    pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
        Module.
    in_channels (int): Input channels.
    channels (int): Channels after modules, before conv_seg.

Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
c                óD   <€ V ^8„  d   QhRS[ S[R3,          RS[RS[RR/# )r5   Úpool_scales.rU  ri  r7   N)rw   r�   )r8   r9   s   "€r,   r:   Ú%BeitPyramidPoolingModule.__annotate__H  s3   ø€ ÷ 

ñ 

¡E©#¨s¨(¥Oð 

Á#ð 

ÑQTð 

ÐY]ñ 

r+   c                ó¾   <€ \         SV `  4        Wn        W n        W0n        \
        P                  ! V Uu. uF  p\        WBVR 7      NK  	  up4      V n        R# u upi ))rh  rU  ri  N)	rt   r?   rz  rU  ri  r   r  rf  Úblocks)rO   rz  rU  ri  rh  r}   s   &&&& €r,   r?   Ú!BeitPyramidPoolingModule.__init__H  sY   ø€ Ü‰ÑÔØ&ÔØ&ÔØ ŒÜ—m’mñ #.óá"-�Jô (°:Ðai×jÙ"-ñó
ˆŽùòs   ¶Ac                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# r!  )rA   r   rx   )r8   r9   s   "€r,   r:   r{  T  s,   ø€ ÷ Sñ S¡U§\¡\ð S±d¹5¿<¹<Õ6Hñ Sr+   c                ó~   € VP                  4       R ,          pV P                   Uu. uF  q3! WR7      NK  	  up# u upi )r  )rZ   )rZ   r}  )rO   rÚ   Úoriginal_sizeÚblocks   &&  r,   ri   Ú BeitPyramidPoolingModule.forwardT  s6   € Ø%×*Ñ*Ó,¨RÕ0ˆØFJÇkÂkÓRÁk¸U��m×8ÁkÑRÐRùÒRs   ¦:)r}  ri  rU  rz  ræ   r²   s   @@r,   rx  rx  ;  s&   ù‡ € ñ
÷

ó 

÷S÷ Sð Sr+   rx  c                   óf   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltV3R lR ltR	tVt	V ;t
# )
ÚBeitUperHeadiY  zê
Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
[UPerNet](https://huggingface.co/papers/1807.10221).

Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
c                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   ÚBeitUperHead.__annotate__a  s   ø€ ÷  
ñ  
™zð  
¨dñ  
r+   c           
     ó  <€ \         SV `  4        VP                  V n        VP                  .^,          V n        VP                  V n        \        P                  ! V P
                  VP                  ^R7      V n	        \        V P                  V P                  R,          V P
                  4      V n        \        V P                  R,          \        V P                  4      V P
                  ,          ,           V P
                  ^^R7      V n        \        P                  ! 4       V n        \        P                  ! 4       V n        V P                  RR  Fo  pV P                   P%                  \        W P
                  ^R7      4       V P"                  P%                  \        V P
                  V P
                  ^^R7      4       Kq  	  \        \        V P                  4      V P
                  ,          V P
                  ^^R7      V n        R# )é   rm  ©rW  rY  NrX   )rt   r?   rz  rC   rU  ri  r   r`  r/  rJ  rx  Úpsp_modulesrS  ÚlenÚpsp_bottleneckr  Úlateral_convsÚ	fpn_convsÚappendÚfpn_bottleneck)rO   r6   rU  r}   s   && €r,   r?   ÚBeitUperHead.__init__a  s|  ø€ Ü‰ÑÔà!×-Ñ-ˆÔØ"×.Ñ.Ð/°!Õ3ˆÔØ×*Ñ*ˆŒÜŸ)š) D§M¡M°6×3DÑ3DÐRSÔTˆŒô 4Ø×ÑØ×Ñ˜RÕ Ø�M‰Mó
ˆÔô
 ,Ø×Ñ˜RÕ ¤3 t×'7Ñ'7Ó#8¸4¿=¹=Õ#HÕHØ�M‰MØØô	
ˆÔô  Ÿ]š]›_ˆÔÜŸš›ˆŒØ×+Ñ+¨C¨RÓ0ˆKØ×Ñ×%Ñ%¤m°KÇÁÐ\]Ô&^Ô_Ø�N‰N×!Ñ!¤-°·±¸t¿}¹}ÐZ[ÐefÔ"gÖhñ 1ô ,Ü�× Ñ Ó! D§M¡MÕ1Ø�M‰MØØô	
ˆÖr+   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# r!  ©rx   rA   r   )r8   r9   s   "€r,   r:   r‡  ƒ  s(   ø€ ÷ 1ñ 1©©e¯l©lÕ);ð 1ÁÇÁñ 1r+   c                óŠ   € VR,          p\         P                  ! V.V P                  V4      O^R7      pV P                  V4      # )r=   rV   rX   )rA   r^   r‹  r�  )rO   rÚ   ru  s   && r,   Úpsp_forwardÚBeitUperHead.psp_forwardƒ  sA   € Ø$ RÕ(ˆÜ—y’y ,Ð!P°×1AÑ1AÀ,Ó1OÐ!PÐVWÔXˆØ×"Ñ" <Ó0Ð0r+   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# ©r5   Úencoder_hidden_statesr7   r”  )r8   r9   s   "€r,   r:   r‡  ˆ  s(   ø€ ÷ ñ ©T±%·,±,Õ-?ð ÁEÇLÁLñ r+   c           	     óÀ  € . p\        V P                  V4       F  w  r4VP                  V! V4      4       K  	  VP                  V P                  V4      4       \	        V4      p\        V^,
          ^ R4       Fi  pW&^,
          ,          P                  R,          pW&^,
          ,          \        P                  P                  W&,          VRRR7      ,           W&^,
          &   Kk  	  . p\        V^,
          4       F1  pVP                  V P                  V,          ! W&,          4      4       K3  	  VP                  VR,          4       \        V^,
          ^ R4       FF  p\        P                  P                  W†,          V^ ,          P                  R,          RRR7      W†&   KH  	  \        P                  ! V^R7      pV P                  V4      p	V P                  V	4      p	V	# )r=   r  rš   Frœ   rV   rX   )ÚziprŽ  r�  r–  rŒ  rþ   rY   r   rŸ   r    r�  rA   r^   r‘  rJ  )
rO   rš  ÚlateralsÚlateral_convru  Úused_backbone_levelsr
  Ú
prev_shapeÚfpn_outsÚoutputs
   &&        r,   ri   ÚBeitUperHead.forwardˆ  s�  € àˆÜ*-¨d×.@Ñ.@ÐBWÖ*XÑ&ˆLØ�O‰O™L¨Ó6Ö7ñ +Yð 	�‰˜×(Ñ(Ð)>Ó?Ô@ô  # 8›}ÐÜÐ+¨aÕ/°°BÖ7ˆAØ! a¥%�×.Ñ.¨rÕ2ˆJØ&¨1¥u�o´·±×0IÑ0IØ• *°:ÈUð 1Jó 1õ ˆH˜•U‹Oñ 8ð ˆÜÐ+¨aÕ/Ö0ˆAØ�O‰O˜DŸN™N¨1Ö-¨h­kÓ:Ö;ñ 1ð 	�‰˜ �Ô%äÐ+¨aÕ/°°BÖ7ˆAÜŸ-™-×3Ñ3Ø• (¨1¥+×"3Ñ"3°BÕ"7¸jÐX]ð 4ó ˆH‹Kñ 8ô —9’9˜X¨1Ô-ˆØ×$Ñ$ XÓ.ˆØ—‘ Ó(ˆàˆr+   )	ri  rJ  r‘  r�  rU  rŽ  rz  r�  r‹  )r%   r&   r'   r(   r)   r?   r–  ri   r*   rl   r±   r²   s   @@r,   r…  r…  Y  s.   ù‡ € ñ÷ 
ó  
÷D1ð 1÷
÷ ð r+   r…  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚBeitFCNHeadiª  a·  
Fully Convolution Networks for Semantic Segmentation. This head is implemented of
[FCNNet](https://huggingface.co/papers/1411.4038>).

Args:
    config (BeitConfig): Configuration.
    in_channels
    kernel_size (int): The kernel size for convs in the head. Default: 3.
    dilation (int): The dilation rate for convs in the head. Default: 1.


Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
c                ó\   <€ V ^8„  d   QhRS[ RS[RS[RS[S[S[S[3,          ,          RR/# )r5   r6   Úin_indexrW  rZ  r7   N)r    r�   rw   )r8   r9   s   "€r,   r:   ÚBeitFCNHead.__annotate__¹  sN   ø€ ÷ !Uñ !UÙ ð!UÙ,/ð!UÙBEð!UÙUXÑ[`ÑadÑfiÐaiÕ[jÕUjð!Uà	ñ!Ur+   c                ól  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        W n
        V^,          V,          p\        P                  ! 4       V n        V P                  ^ 8”  d�   V P                  P                  \        V P                  V P
                  W5VR7      4       \!        V P                  ^,
          4       F@  pV P                  P                  \        V P
                  V P
                  VVVR7      4       KB  	  V P                  '       d@   \        V P                  V P
                  ,           V P
                  W3^,          R7      V n        \        P$                  ! V P
                  VP&                  ^R7      V n        R# )r5   )rW  rY  rZ  rŠ  rm  N)rt   r?   rC   rU  Úauxiliary_channelsri  Úauxiliary_num_convsÚ	num_convsÚauxiliary_concat_inputÚconcat_inputr§  r   r  Úconvsr�  rS  rþ   Úconv_catr`  r/  rJ  )rO   r6   r§  rW  rZ  Úconv_paddingr`   r}   s   &&&&&  €r,   r?   ÚBeitFCNHead.__init__¹  sG  ø€ ô 	‰ÑÔØ!×-Ñ-ˆÔØ×1Ñ1ˆŒØ×3Ñ3ˆŒØ"×9Ñ9ˆÔØ Œà# qÕ(¨HÕ4ˆÜ—]’]“_ˆŒ
Ø�>‰>˜AÔØ�J‰J×ÑÜØ×$Ñ$ d§m¡mÀÐmuôôô
 ˜4Ÿ>™>¨AÕ-Ö.�Ø—
‘
×!Ñ!Ü!ØŸ™ØŸ™Ø$/Ø ,Ø!)ôöñ /ð ××ÐÜ)Ø× Ñ  4§=¡=Õ0°$·-±-È[ÐqrÕbrôˆDŒMô Ÿ)š) D§M¡M°6×3DÑ3DÐRSÔTˆŽr+   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# r™  r”  )r8   r9   s   "€r,   r:   r¨  Ü  s(   ø€ ÷ ñ ©T±%·,±,Õ-?ð ÁEÇLÁLñ r+   c                óú   € WP                   ,          pTpV P                   F  pV! V4      pK  	  V P                  '       d)   V P                  \        P
                  ! W#.^R7      4      pV P                  V4      pV# )r=   rV   )r§  r¯  r®  r°  rA   r^   rJ  )rO   rš  rã   rÚ   rp  s   &&   r,   ri   ÚBeitFCNHead.forwardÜ  sh   € Ø(¯©Õ7ˆØ ˆØ—J”JˆDÙ  Ó/ŠMñ à××ÐØ ŸM™M¬%¯)ª)°XÐ4MÐSTÔ*UÓVˆMØŸ™¨Ó6ˆØÐr+   )ri  rJ  r®  r°  r¯  rU  r§  r¬  )r5   rc  r=   ræ   r²   s   @@r,   r¥  r¥  ª  s&   ù‡ € ñ÷!Uõ !U÷F÷ ð r+   r¥  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚBeitFPNUpBlockiç  uE   4x upsampling block: ConvTranspose â†’ BN â†’ GELU â†’ ConvTranspose.c                ó0   <€ V ^8„  d   QhRS[ RS[ RS[ RR/# )r5   rC   rW  rX  r7   Nrj  )r8   r9   s   "€r,   r:   ÚBeitFPNUpBlock.__annotate__ê  s0   ø€ ÷ tñ t¡Cð t±cð tÁsð tÐSWñ tr+   c                ó  <€ \         SV `  4        \        P                  ! WW#R 7      V n        \        P
                  ! V4      V n        \        P                  ! 4       V n        \        P                  ! WW#R 7      V n	        R# )©rW  rX  N)
rt   r?   r   ÚConvTranspose2dÚconv_transpose1ÚBatchNorm2dÚnormalizationÚGELUr\  Úconv_transpose2)rO   rC   rW  rX  r}   s   &&&&€r,   r?   ÚBeitFPNUpBlock.__init__ê  sX   ø€ Ü‰ÑÔÜ!×1Ò1°+ÐXcÔsˆÔÜŸ^š^¨KÓ8ˆÔÜŸ'š'›)ˆŒÜ!×1Ò1°+ÐXcÔsˆÖr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r!  r"  )r8   r9   s   "€r,   r:   r¹  ñ  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r+   c                óŽ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# rk   )r½  r¿  r\  rÁ  r&  s   &&r,   ri   ÚBeitFPNUpBlock.forwardñ  sF   € Ø×,Ñ,¨]Ó;ˆØ×*Ñ*¨=Ó9ˆØŸ™¨Ó6ˆØ×,Ñ,¨]Ó;ˆØÐr+   )r\  r½  rÁ  r¿  )r5   r5   ræ   r²   s   @@r,   r·  r·  ç  s#   ù‡ € ÙO÷tõ t÷÷ ð r+   r·  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚBeitFPNNeckiù  z˜
4-level feature pyramid neck for BeiT. Produces x4 upsample, x2 upsample,
identity, and x2 downsample outputs from the four selected ViT feature maps.
c                ó    <€ V ^8„  d   QhRS[ /# r¶   r   )r8   r9   s   "€r,   r:   ÚBeitFPNNeck.__annotate__ÿ  s   ø€ ÷ :ñ :™zñ :r+   c                óø   <€ \         SV `  4        \        VP                  4      V n        \
        P                  ! VP                  VP                  ^^R7      V n        \
        P                  ! ^^R7      V n	        R# )r5   r»  N)
rt   r?   r·  rC   Úfpn1r   r¼  Úfpn2Ú	MaxPool2dÚfpn4rÀ   s   &&€r,   r?   ÚBeitFPNNeck.__init__ÿ  sX   ø€ Ü‰ÑÔÜ" 6×#5Ñ#5Ó6ˆŒ	Ü×&Ò& v×'9Ñ'9¸6×;MÑ;MÐ[\ÐefÔgˆŒ	Ü—L’L¨Q°qÔ9ˆŽ	r+   c                óv   <€ V ^8„  d   QhRS[ S[P                  R3,          RS[ S[P                  R3,          /# )r5   Úfeature_maps.r7   )rw   rA   r   )r8   r9   s   "€r,   r:   rÉ    s8   ø€ ÷ 
ñ 
¡E©%¯,©,¸Ð*;Õ$<ð 
ÁÁuÇ|Á|ÐUXÐGXÕAYñ 
r+   c                ó    € V P                  V^ ,          4      V P                  V^,          4      V^,          V P                  V^,          4      3# )rs   ©rË  rÌ  rÎ  )rO   rÑ  s   &&r,   ri   ÚBeitFPNNeck.forward  sC   € à�I‰I�l 1•oÓ&Ø�I‰I�l 1•oÓ&Ø˜�OØ�I‰I�l 1•oÓ&ð	
ð 	
r+   rÓ  ræ   r²   s   @@r,   rÇ  rÇ  ù  s#   ù‡ € ñ÷
:ó :÷
÷ 
ð 
r+   rÇ  c                   ór   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tRt	Vt
V ;t# )ÚBeitForSemanticSegmentationi  c                ó$   <€ V ^8„  d   QhRS[ RR/# r4   r   )r8   r9   s   "€r,   r:   Ú(BeitForSemanticSegmentation.__annotate__  s   ø€ ÷ ñ ™zð ¨dñ r+   c                ór  <€ \         SV `  V4       VP                  V n        \        VR R7      V n        \        V P                  P                  4      ^8w  d   \        R4      h\        V4      V n
        \        V4      V n        VP                  '       d   \        V4      MRV n        V P!                  4        R# )Fr.  zÇBeitForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.N)rt   r?   r/  rõ   r0  rŒ  r6   Úout_indicesÚ
ValueErrorrÇ  Úfpnr…  Údecode_headÚuse_auxiliary_headr¥  Úauxiliary_headr	  rÀ   s   &&€r,   r?   Ú$BeitForSemanticSegmentation.__init__  s˜   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô>ˆŒ	äˆt�{‰{×&Ñ&Ó'¨1Ô,Üð-óð ô
 ˜vÓ&ˆŒô (¨Ó/ˆÔØ5;×5N×5NÐ5Nœk¨&Ô1ÐTXˆÔð 	�‰Ör+   c                óœ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[RS[S[,          RS[S[,          /# rM  )rA   r   r˜   r   r   rw   r   )r8   r9   s   "€r,   r:   rØ  (  sb   ø€ ÷ G
ñ G
á—l‘l TÕ)ðG
ñ —‘˜tÕ#ðG
ñ #'ð	G
ñ
 Ñ+Õ,ðG
ñ 
Ñ(Õ	(ñG
r+   c                óD  aaaa€ Ve'   V P                   P                  ^8X  d   \        R4      hV P                  ! V3RV/VB pVP                  oVP
                  w  orgpWpP                   P                  ,          oW€P                   P                  ,          o\        ;QJ d.    . VVVV3R lV P                   P                   4       F  NK  	  5M'! VVVV3R lV P                   P                   4       4      p	V P                  V	4      p	V P                  V	4      p
RpV P                  e   V P                  V	4      pRpVe?   V P                  V
VV P                   P                  VV P                   P                  R7      p\        VV
VP                  VP                   R7      # )a¼  
labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
    Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, BeitForSemanticSegmentation
>>> from PIL import Image
>>> import requests

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")
>>> model = BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")

>>> inputs = image_processor(images=image, return_tensors="pt")
>>> outputs = model(**inputs)
>>> # logits are of shape (batch_size, num_labels, height, width)
>>> logits = outputs.logits
```Nz/The number of labels should be greater than oner_   c              3   ó–   <"  € T F>  pSV^,
          ,          R,          P                  ^^4      P                  SRSS4      x € K@  	  R# 5i)r=   Nr>  rX   )Ú	transposerž   )Ú.0r
  rd   rš  Úpatch_heightÚpatch_widths   & €€€€r,   Ú	<genexpr>Ú6BeitForSemanticSegmentation.forward.<locals>.<genexpr>U  sJ   øé € ð 
á,�ð " ! a¥%Õ(¨Õ/×9Ñ9¸!¸QÓ?×GÑGÈ
ÐTVÐXdÐfq×rÐrÛ,ùs   ƒAA	)Úignore_indexÚauxiliary_logitsÚauxiliary_loss_weightr:  )r6   r/  rÛ  r0  rÚ   rY   rH   rw   rÚ  rÜ  rÝ  rß  rO  Úsemantic_loss_ignore_indexrì  r   r=  )rO   rR   r8  r_   rÝ   r@  r`   ra   rb   rÑ  r<  rë  r;  rd   rš  ræ  rç  s   &&&&,        @@@@r,   ri   Ú#BeitForSemanticSegmentation.forward%  s}  û€ ðB Ò $§+¡+×"8Ñ"8¸AÔ"=ÜÐNÓOÐOØ—)’)Øñ
à%=ð
ð ñ
ˆð !(× 5Ñ 5ÐØ'3×'9Ñ'9Ñ$ˆ
�A˜uØ§¡×!7Ñ!7Õ7ˆØŸ{™{×5Ñ5Õ5ˆ÷ ”u÷ 
à—[‘[×,Ò,ó
—u‘u÷ 
à—[‘[×,Ò,ó
ó 
ˆð —x‘x Ó-ˆà×!Ñ! ,Ó/ˆàÐØ×ÑÒ*Ø#×2Ñ2°<Ó@ÐàˆØÒØ×%Ñ%ØØØ!Ÿ[™[×CÑCØ!1Ø&*§k¡k×&GÑ&Gð &ó ˆDô 'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )rß  r0  rÝ  rÜ  r/  rQ  ©r%   r&   r'   r(   r?   r   r   r   ri   r*   rl   r±   r²   s   @@r,   rÖ  rÖ    sA   ù‡ € ÷ó ð* Ø Ø÷G
ñ G
ó ó !ó ÷G
ð G
r+   rÖ  zM
    BEiT backbone, to be used with frameworks like DETR and MaskFormer.
    c                   ób   a a€ ] tR tRt oV 3R lt]]]V3R lR l4       4       4       tRt	Vt
V ;t# )ÚBeitBackboneir  c                óV  <€ \         SV `  V4       \        VP                  ^,           4       Uu. uF  q!P                  NK  	  upV n        \        VRR7      V n        VP                  '       d   \        V4      M\        P                  ! 4       V n        V P                  4        R# u upi )r=   Fr.  N)rt   r?   rþ   rÿ   rC   Únum_featuresrõ   r0  Úadd_fpnrÇ  r   rÐ   rÜ  r	  )rO   r6   r`   r}   s   && €r,   r?   ÚBeitBackbone.__init__x  sz   ø€ Ü‰Ñ˜Ô ä9>¸v×?WÑ?WÐZ[Õ?[Ô9\Ó]Ñ9\°A×/Ô/Ñ9\Ñ]ˆÔÜ˜f¸Ô>ˆŒ	Ø*0¯.¯.¨.”;˜vÔ&¼b¿kºk»mˆŒð 	�‰Öùò ^s   ¯B&c                ó<   <€ V ^8„  d   QhRS[ RS[S[,          RS[/# )r5   rR   rÝ   r7   )r   r   r   r	   )r8   r9   s   "€r,   r:   ÚBeitBackbone.__annotate__…  s/   ø€ ÷ 3
ñ 3
áð3
ñ Ñ+Õ,ð3
ñ 
ñ	3
r+   c                ó@  € VP                   w  r4rVWPP                  P                  ,          pW`P                  P                  ,          pV P                  ! V3/ VB p	V	P                  p
Rp\        V P                  V
4       Fj  w  rÍWÀP                  9   g   K  V P                  P                  '       d/   VR,          pVP                  ^^4      pVP                  VRWx4      pW½3,          pKl  	  V P                  V4      p\        VV	P                  V	P                  R7      # )aª  
Examples:

```python
>>> from transformers import AutoImageProcessor, AutoBackbone
>>> import torch
>>> from PIL import Image
>>> import requests

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
>>> model = AutoBackbone.from_pretrained(
...     "microsoft/beit-base-patch16-224", out_features=["stage1", "stage2", "stage3", "stage4"]
... )

>>> inputs = processor(image, return_tensors="pt")

>>> outputs = model(**inputs)
>>> feature_maps = outputs.feature_maps
>>> list(feature_maps[-1].shape)
[1, 768, 14, 14]
```)rÑ  rÚ   r=  r$   r$  rX   )rY   r6   rH   r0  rÚ   rœ  Ústage_namesÚout_featuresÚreshape_hidden_statesrä  rž   rÜ  r	   r=  )rO   rR   rÝ   rd   r`   ra   rb   ræ  rç  r@  rÚ   rÑ  Ústageru  s   &&,           r,   ri   ÚBeitBackbone.forward‚  sÿ   € ð@ (4×'9Ñ'9Ñ$ˆ
�vØ§¡×!7Ñ!7Õ7ˆØŸ{™{×5Ñ5Õ5ˆØ—)’)˜LÑ3¨FÑ3ˆà×-Ñ-ˆØˆÜ#& t×'7Ñ'7¸Ö#GÑˆEØ×)Ñ)Ö)Ø—;‘;×4×4Ð4Ø#/°Õ#9�LØ#/×#9Ñ#9¸!¸QÓ#?�LØ#/×#7Ñ#7¸
ÀBÈÓ#b�Là Õ/’ñ $Hð —x‘x Ó-ˆäØ%Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )r0  rÜ  ró  rï  r²   s   @@r,   rñ  rñ  r  s3   ù‡ € õð Ø Ø÷3
ó ó !ó ÷3
ð 3
r+   rñ  )rF  r*  rÖ  rõ   rè   rñ  )Hr)   Údataclassesr   rA   r   r   Ú r   rë   Úbackbone_utilsr   r   Úmasking_utilsr   Úmodeling_outputsr	   r
   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úresnet.modeling_resnetr   Úswin.modeling_swinr   Úvit.modeling_vitr   r   r   r   r   r   Úconfiguration_beitr    r#   r.   r2   r>   ro   r´   rÄ   rÆ   rÈ   rè   rõ   r  r*  rF  rS  rf  rx  r…  r¥  r·  rÇ  rÖ  rñ  Ú__all__r$   r+   r,   Ú<module>r     sF  ðñ å !ã ß å &ß HÝ 6÷õ õ .Ý &Ý @ß BÑ Bß IÝ 5Ý 4Ý -ß t× tÝ *ñ ðôð
 ôÐ!;ó ó óðô	Ð,ô 	ô'�]ô 'ôTV3˜rŸy™yô V3ôr`�Lô `ô	ˆfô 	ô	�<ô 	ô7�ô 7ðt ôTÐ,ó Tó ðTð. ôJjÐ#ó Jjó ðJjôZ	v�—‘ô 	vñ ðôôS
Ð!4ó S
óðS
ñl ðôô/
Ð!4ó /
óð/
ôd
�Oô 
ô4
˜bŸi™iô 
ôS˜rŸy™yô Sô<N�2—9‘9ô Nôb:�"—)‘)ô :ôz�R—Y‘Yô ô$
�"—)‘)ô 
ð* ô`
Ð"5ó `
ó ð`
ñF ðôô
A
�=Ð"5ó A
óð
A
òH‚r+   