+
    œ°³jÐV  ã                   ó†  € R t ^ RIt^ RIt^ RIHt ^ RIt^ RIHu Ht	 ^ RIHt ^RI
Ht ^RIHt ^RIHtHt ^RIHt ^RIHtHt ^R	IHt ]P2                  ! ]4      t ! R
 R]P8                  4      t ! R R]P8                  4      t ! R R]P8                  4      t ! R R]P8                  4      t  ! R R]P8                  4      t! ! R R]P8                  4      t" ! R R]P8                  4      t# ! R R]P8                  4      t$] ! R R]4      4       t%] ! R R]%4      4       t&]! RR7       ! R  R!]%4      4       t'. R"Ot(R# )#zPyTorch PVT model.N)ÚIterable)Únn)Úinitialization)ÚACT2FN)ÚBaseModelOutputÚImageClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚlogging©Ú	PvtConfigc                   ój   a a€ ] tR t^%t oRtR	V3R lV 3R llltV3R lR ltV3R lR ltRtVt	V ;t
# )
ÚPvtPatchEmbeddingszÜ
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                ó„   <€ V ^8„  d   QhRS[ RS[S[S[,          ,          RS[S[S[,          ,          RS[RS[RS[RS[/# )é   ÚconfigÚ
image_sizeÚ
patch_sizeÚstrideÚnum_channelsÚhidden_sizeÚ	cls_token)r   Úintr   Úbool)ÚformatÚ__classdict__s   "€Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/pvt/modeling_pvt.pyÚ__annotate__ÚPvtPatchEmbeddings.__annotate__,   sl   ø€ ÷ @ñ @áð@ñ ™(¡3�-Õ'ð@ñ ™(¡3�-Õ'ð	@ñ
 ð@ñ ð@ñ ð@ñ ñ@ó    c                óN  <€ \         S	V `  4        Wn        \        V\        P
                  P                  4      '       d   TMW"3p\        V\        P
                  P                  4      '       d   TMW33pV^,          V^,          ,          V^ ,          V^ ,          ,          ,          pW n        W0n        WPn	        W€n
        \        P                  ! \        P                  ! ^V'       d
   V^,           MTV4      4      V n        V'       d-   \        P                  ! \        P                   ! ^^V4      4      MRV n        \        P$                  ! WVWCR7      V n        \        P(                  ! WaP*                  R7      V n        \        P.                  ! VP0                  R7      V n        R# )é   N©Úkernel_sizer   ©Úeps)Úp)ÚsuperÚ__init__r   Ú
isinstanceÚcollectionsÚabcr   r   r   r   Únum_patchesr   Ú	ParameterÚtorchÚrandnÚposition_embeddingsÚzerosr   ÚConv2dÚ
projectionÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout)
Úselfr   r   r   r   r   r   r   r,   Ú	__class__s
   &&&&&&&& €r   r(   ÚPvtPatchEmbeddings.__init__,   s  ø€ ô 	‰ÑÔØŒÜ#-¨j¼+¿/¹/×:RÑ:R×#SÒ#S‘ZÐZdÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:R×#SÒ#S‘ZÐZdÐYqˆ
Ø! !•}¨
°1­Õ5¸*ÀQ½-È:ÐVWÍ=Õ:XÕYˆØ$ŒØ$ŒØ(ÔØ&Ôä#%§<¢<Ü�KŠK˜¯i˜;¨ž?¸[È+ÓVó$
ˆÔ ÷ JSœŸš¤e§k¢k°!°Q¸Ó&DÔEÐX\ˆŒÜŸ)š) LÈ6ÔeˆŒÜŸ,š, {×8MÑ8MÔNˆŒÜ—z’z F×$>Ñ$>Ô?ˆŽr   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r   Ú
embeddingsÚheightÚwidthÚreturn)r.   ÚTensorr   )r   r   s   "€r   r   r   H   s5   ø€ ÷ 	'ñ 	'±5·<±<ð 	'Éð 	'ÑUXð 	'Ñ]b×]iÑ]iñ 	'r   c                ó²  € W#,          p\         P                  P                  4       '       gB   W@P                  P                  V P                  P                  ,          8X  d   V P
                  # VP                  ^W#R4      P                  ^ ^^^4      p\        P                  ! WV3RR7      pVP                  ^RW#,          4      P                  ^ ^^4      pV# )r!   Úbilinear)ÚsizeÚmodeéÿÿÿÿ)
r.   ÚjitÚ
is_tracingr   r   r0   ÚreshapeÚpermuteÚFÚinterpolate)r:   r>   r?   r@   r,   Úinterpolated_embeddingss   &&&&  r   Úinterpolate_pos_encodingÚ+PvtPatchEmbeddings.interpolate_pos_encodingH   sµ   € Ø•nˆô �y‰y×#Ñ#×%Ò%¨+¿¹×9OÑ9OÐRV×R]ÑR]×RhÑRhÕ9hÔ*hØ×+Ñ+Ð+Ø×'Ñ'¨¨6¸"Ó=×EÑEÀaÈÈAÈqÓQˆ
Ü"#§-¢-°
È%ÀÐWaÔ"bÐØ"9×"AÑ"AÀ!ÀRÈÍÓ"X×"`Ñ"`ÐabÐdeÐghÓ"iÐØ&Ð&r   c                óh   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[S[3,          /# )r   Úpixel_valuesrA   )r.   rB   Útupler   )r   r   s   "€r   r   r   S   s/   ø€ ÷ )ñ )¡E§L¡Lð )±U¹5¿<¹<ÉÉcÐ;QÕ5Rñ )r   c                óŽ  € VP                   w  r#rEW0P                  8w  d   \        R 4      hV P                  V4      pVP                   Ev rtpVP	                  ^4      P                  ^^4      pV P                  V4      pV P                  e†   V P                  P                  VRR4      p	\        P                  ! W˜3^R7      pV P                  V P                  R,          WE4      p
\        P                  ! V P                  R,          V
3^R7      p
MV P                  V P                  WE4      p
V P                  WŠ,           4      pW„V3# )zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.©ÚdimrG   )ºNNNºr!   NN)rW   :Nr!   N)Úshaper   Ú
ValueErrorr3   ÚflattenÚ	transposer6   r   Úexpandr.   ÚcatrO   r0   r9   )r:   rR   Ú
batch_sizer   r?   r@   Úpatch_embedÚ_r>   r   r0   s   &&         r   ÚforwardÚPvtPatchEmbeddings.forwardS   s'  € Ø2>×2DÑ2DÑ/ˆ
 &Ø×,Ñ,Ô,ÜØwóð ð —o‘o lÓ3ˆØ'×-Ñ-Ñˆ�EØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆØ—_‘_ [Ó1ˆ
Ø�>‰>Ò%ØŸ™×-Ñ-¨j¸"¸bÓAˆIÜŸš IÐ#:ÀÔBˆJØ"&×"?Ñ"?À×@XÑ@XÐY^Õ@_ÐagÓ"oÐÜ"'§)¢)¨T×-EÑ-EÀeÕ-LÐNaÐ,bÐhiÔ"jÑà"&×"?Ñ"?À×@XÑ@XÐZ`Ó"hÐØ—\‘\ *Õ"BÓCˆ
à 5Ð(Ð(r   )
r   r   r9   r   r6   r   r,   r   r0   r3   ©F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r(   rO   rb   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r;   r   s   @@r   r   r   %   s/   ù‡ € ñ÷@õ @÷8	'ð 	'÷)÷ )ð )r   r   c                   óP   a a€ ] tR t^it oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPvtSelfOutputc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r   r   ©r   r   )r   r   s   "€r   r   ÚPvtSelfOutput.__annotate__j   s   ø€ ÷ >ñ >™yð >±sñ >r   c                ó¤   <€ \         SV `  4        \        P                  ! W"4      V n        \        P
                  ! VP                  4      V n        R # ©N)r'   r(   r   ÚLinearÚdenser7   r8   r9   )r:   r   r   r;   s   &&&€r   r(   ÚPvtSelfOutput.__init__j   s4   ø€ Ü‰ÑÔÜ—Y’Y˜{Ó8ˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úhidden_statesrA   ©r.   rB   )r   r   s   "€r   r   rr   o   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r   c                óJ   € V P                  V4      pV P                  V4      pV# rt   ©rv   r9   ©r:   rz   s   &&r   rb   ÚPvtSelfOutput.forwardo   s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr   r}   ©	re   rf   rg   rh   r(   rb   rj   rk   rl   rm   s   @@r   ro   ro   i   s   ù‡ € ÷>ó >÷
÷ ð r   ro   c                   ój   a a€ ] tR t^ut oRtV3R lV 3R lltV3R lR ltR	V3R lR lltRtVt	V ;t
# )
ÚPvtEfficientSelfAttentionzxEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://huggingface.co/papers/2102.12122).c                ó2   <€ V ^8„  d   QhRS[ RS[RS[RS[/# ©r   r   r   Únum_attention_headsÚsequences_reduction_ratio©r   r   Úfloat)r   r   s   "€r   r   Ú&PvtEfficientSelfAttention.__annotate__x   s3   ø€ ÷ Sñ SÙðSÙ.1ðSÙHKðSÙhmñSr   c                ó¼  <€ \         SV `  4        W n        W0n        V P                  V P                  ,          ^ 8w  d'   \	        RV P                   RV P                   R24      h\        V P                  V P                  ,          4      V n        V P                  V P                  ,          V n        \        P                  ! V P                  V P                  VP                  R7      V n        \        P                  ! V P                  V P                  VP                  R7      V n        \        P                  ! V P                  V P                  VP                  R7      V n        \        P                  ! VP                  4      V n        W@n        V^8”  dF   \        P$                  ! W"WDR7      V n        \        P(                  ! W!P*                  R7      V n        R# R# )é    zThe hidden size (z6) is not a multiple of the number of attention heads (Ú))Úbiasr"   r$   N)r'   r(   r   r…   rZ   r   Úattention_head_sizeÚall_head_sizer   ru   Úqkv_biasÚqueryÚkeyÚvaluer7   Úattention_probs_dropout_probr9   r†   r2   Úsequence_reductionr4   r5   r6   ©r:   r   r   r…   r†   r;   s   &&&&&€r   r(   Ú"PvtEfficientSelfAttention.__init__x   se  ø€ ô 	‰ÑÔØ&ÔØ#6Ô à×Ñ˜d×6Ñ6Õ6¸!Ô;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°1ð6óð ô
 $' t×'7Ñ'7¸$×:RÑ:RÕ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—Y’Y˜t×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒ
Ü—9’9˜T×-Ñ-¨t×/AÑ/AÈÏÉÔXˆŒÜ—Y’Y˜t×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒ
ä—z’z &×"EÑ"EÓFˆŒà)BÔ&Ø$ qÔ(Ü&(§i¢iØÐ6Oô'ˆDÔ#ô !Ÿlšl¨;×<QÑ<QÔRˆDŽOñ	 )r   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# ry   )r   r.   rB   )r   r   s   "€r   r   r‰   •   s   ø€ ÷ 1ñ 1±#ð 1¹%¿,¹,ñ 1r   c                ó¬   € VP                  4       R R V P                  V P                  3,           pVP                  V4      pVP	                  ^ ^^^4      # )NrG   )rE   r…   rŽ   ÚviewrK   )r:   rz   Ú	new_shapes   && r   Útranspose_for_scoresÚ.PvtEfficientSelfAttention.transpose_for_scores•   sT   € Ø!×&Ñ&Ó(¨¨"Ð-°×1IÑ1IÈ4×KcÑKcÐ0dÕdˆ	Ø%×*Ñ*¨9Ó5ˆØ×$Ñ$ Q¨¨1¨aÓ0Ð0r   c                óp   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[RS[S[ P                  ,          /# ©r   rz   r?   r@   Úoutput_attentionsrA   ©r.   rB   r   r   rS   )r   r   s   "€r   r   r‰   š   sJ   ø€ ÷ *ñ *á—|‘|ð*ñ ð*ñ ð	*ñ
  ð*ñ 
‰u�|‰|Õ	ñ*r   c                óÐ  € V P                  V P                  V4      4      pV P                  ^8”  dx   VP                  w  rgpVP	                  ^ ^^4      P                  WhW#4      pV P                  V4      pVP                  WhR4      P	                  ^ ^^4      pV P                  V4      pV P                  V P                  V4      4      p	V P                  V P                  V4      4      p
\        P                  ! WYP                  RR4      4      pV\        P                  ! V P                  4      ,          p\         P"                  P%                  VRR7      pV P'                  V4      p\        P                  ! WÊ4      pVP	                  ^ ^^^4      P)                  4       pVP+                  4       RR V P,                  3,           pVP/                  V4      pV'       d   WÜ3pV# V3pV# )r!   rU   NrG   éþÿÿÿ)rœ   r‘   r†   rY   rK   rJ   r•   r6   r’   r“   r.   Úmatmulr\   ÚmathÚsqrtrŽ   r   Ú
functionalÚsoftmaxr9   Ú
contiguousrE   r�   rš   )r:   rz   r?   r@   r    Úquery_layerr_   Úseq_lenr   Ú	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss   &&&&&           r   rb   Ú!PvtEfficientSelfAttention.forwardš   s·  € ð ×/Ñ/°·
±
¸=Ó0IÓJˆà×)Ñ)¨AÔ-Ø0=×0CÑ0CÑ-ˆJ à)×1Ñ1°!°Q¸Ó:×BÑBÀ:Ð]cÓkˆMà ×3Ñ3°MÓBˆMà)×1Ñ1°*ÈBÓO×WÑWÐXYÐ[\Ð^_Ó`ˆMØ ŸO™O¨MÓ:ˆMà×-Ñ-¨d¯h©h°}Ó.EÓFˆ	Ø×/Ñ/°·
±
¸=Ó0IÓJˆô !Ÿ<š<¨×5HÑ5HÈÈRÓ5PÓQÐà+¬d¯iªi¸×8PÑ8PÓ.QÕQÐô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸš _ÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÕ"SÐØ%×*Ñ*Ð+BÓCˆç6G�=Ð2ˆàˆð O\ÐM]ˆàˆr   )r�   rŽ   r9   r   r’   r6   r…   r‘   r•   r†   r“   rd   )re   rf   rg   rh   ri   r(   rœ   rb   rj   rk   rl   rm   s   @@r   r‚   r‚   u   s0   ù‡ € ñ C÷Só S÷:1ð 1÷
*÷ *ò *r   r‚   c                   óT   a a€ ] tR t^Çt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPvtAttentionc                ó2   <€ V ^8„  d   QhRS[ RS[RS[RS[/# r„   r‡   )r   r   s   "€r   r   ÚPvtAttention.__annotate__È   s3   ø€ ÷ 
Eñ 
EÙð
EÙ.1ð
EÙHKð
EÙhmñ
Er   c                ón   <€ \         SV `  4        \        VVVVR 7      V n        \	        WR7      V n        R# ))r   r…   r†   )r   N)r'   r(   r‚   r:   ro   Úoutputr–   s   &&&&&€r   r(   ÚPvtAttention.__init__È   s6   ø€ ô 	‰ÑÔÜ-ØØ#Ø 3Ø&?ô	
ˆŒ	ô $ FÔDˆŽr   c                óp   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[RS[S[ P                  ,          /# rŸ   r¡   )r   r   s   "€r   r   r·   Ô   sB   ø€ ÷ ñ Ù"Ÿ\™\ðÙ36ðÙ?BðÙW[ðá	‰u�|‰|Õ	ñr   c                ó|   € V P                  WW44      pV P                  V^ ,          4      pV3VR,          ,           pV# )r‹   rX   )r:   r¹   )r:   rz   r?   r@   r    Úself_outputsÚattention_outputr²   s   &&&&&   r   rb   ÚPvtAttention.forwardÔ   s>   € ð —y‘y ¸ÓQˆàŸ;™; |°A¥Ó7ÐØ#Ð%¨°RÕ(8Õ8ˆØˆr   )r¹   r:   rd   r€   rm   s   @@r   rµ   rµ   Ç   s    ù‡ € ÷
Eó 
E÷÷ ò r   rµ   c                   óT   a a€ ] tR t^Þt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚPvtFFNc          	      óN   <€ V ^8„  d   QhRS[ RS[RS[R,          RS[R,          /# )r   r   Úin_featuresÚhidden_featuresNÚout_featuresrq   )r   r   s   "€r   r   ÚPvtFFN.__annotate__ß   s;   ø€ ÷ >ñ >áð>ñ ð>ñ ˜t�ð	>ñ
 ˜D•jñ>r   c                ó†  <€ \         SV `  4        Ve   TMTp\        P                  ! W#4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n	        MVP                  V n	        \        P                  ! W44      V n
        \        P                  ! VP                  4      V n        R # rt   )r'   r(   r   ru   Údense1r)   Ú
hidden_actÚstrr   Úintermediate_act_fnÚdense2r7   r8   r9   )r:   r   rÃ   rÄ   rÅ   r;   s   &&&&&€r   r(   ÚPvtFFN.__init__ß   s†   ø€ ô 	‰ÑÔØ'3Ò'?‘|À[ˆÜ—i’i Ó=ˆŒÜ�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÕ$à'-×'8Ñ'8ˆDÔ$Ü—i’i Ó>ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ry   r{   )r   r   s   "€r   r   rÆ   ð   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r   c                ó°   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# rt   )rÈ   rË   r9   rÌ   r~   s   &&r   rb   ÚPvtFFN.forwardð   sP   € ØŸ™ MÓ2ˆØ×0Ñ0°Ó?ˆØŸ™ ]Ó3ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr   )rÈ   rÌ   r9   rË   )NNr€   rm   s   @@r   rÁ   rÁ   Þ   s   ù‡ € ÷>õ >÷"÷ ð r   rÁ   c                   ój   a a€ ] tR t^út oRtR	V3R lV 3R llltV3R lR ltV3R lR ltRtVt	V ;t
# )
ÚPvtDropPathzÃStochastic depth (DropPath) per sample, for residual blocks.

Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
<https://arxiv.org/abs/1603.09382>`_.
c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   Ú	drop_probrA   N)rˆ   )r   r   s   "€r   r   ÚPvtDropPath.__annotate__  s   ø€ ÷ #ñ #¡%ð #°$ñ #r   c                ó0   <€ \         SV `  4        Wn        R # rt   )r'   r(   rÔ   )r:   rÔ   r;   s   &&€r   r(   ÚPvtDropPath.__init__  s   ø€ Ü‰ÑÔØ"Žr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ry   r{   )r   r   s   "€r   r   rÕ     s#   ø€ ÷ <ñ <¡U§\¡\ð <±e·l±lñ <r   c                ó˜  € V P                   R 8X  g   V P                  '       g   V# ^V P                   ,
          pVP                  ^ ,          3RVP                  ^,
          ,          ,           p\        P
                  ! W1P                  VP                  R7      p\        P                  ! WB,           4      pVP                  V4      V,          # )ç        )ÚdtypeÚdevice)r!   )
rÔ   ÚtrainingrY   Úndimr.   ÚrandrÛ   rÜ   ÚfloorÚdiv)r:   rz   Ú	keep_probrY   Úrandom_tensors   &&   r   rb   ÚPvtDropPath.forward  s•   € Ø�>‰>˜SÔ ¨¯¯¨Ø Ð Ø˜Ÿ™Õ&ˆ	Ø×$Ñ$ QÕ'Ð)¨D°M×4FÑ4FÈÕ4JÕ,KÕKˆÜŸ
š
 5×0CÑ0CÈM×L`ÑL`ÔaˆÜŸš MÕ$=Ó>ˆØ× Ñ  Ó+¨mÕ;Ð;r   c                ó    <€ V ^8„  d   QhRS[ /# )r   rA   )rÊ   )r   r   s   "€r   r   rÕ     s   ø€ ÷ %ñ %™Cñ %r   c                ó    € R V P                    2# )zp=©rÔ   )r:   s   &r   Ú
extra_reprÚPvtDropPath.extra_repr  s   € Ø�D—N‘NÐ#Ð$Ð$r   rç   )rÚ   )re   rf   rg   rh   ri   r(   rb   rè   rj   rk   rl   rm   s   @@r   rÒ   rÒ   ú   s-   ù‡ € ñ÷#õ #÷<ð <÷%÷ %ð %r   rÒ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚPvtLayeri  c                ó>   <€ V ^8„  d   QhRS[ RS[RS[RS[RS[RS[/# )r   r   r   r…   Ú	drop_pathr†   Ú	mlp_ratior‡   )r   r   s   "€r   r   ÚPvtLayer.__annotate__  sO   ø€ ÷ cñ cáðcñ ðcñ !ð	cñ
 ðcñ $)ðcñ ñcr   c                ó‚  <€ \         SV `  4        \        P                  ! W!P                  R 7      V n        \        VVVVR7      V n        VR8”  d   \        V4      M\        P                  ! 4       V n
        \        P                  ! W!P                  R 7      V n        \        W&,          4      p\        WVR7      V n        R# )r$   )r   r   r…   r†   rÚ   )r   rÃ   rÄ   N)r'   r(   r   r4   r5   Úlayer_norm_1rµ   Ú	attentionrÒ   ÚIdentityrí   Úlayer_norm_2r   rÁ   Úmlp)	r:   r   r   r…   rí   r†   rî   Úmlp_hidden_sizer;   s	   &&&&&&& €r   r(   ÚPvtLayer.__init__  sŽ   ø€ ô 	‰ÑÔÜŸLšL¨×:OÑ:OÔPˆÔÜ%ØØ#Ø 3Ø&?ô	
ˆŒð 4=¸s´?œ YÔ/ÌÏÊËˆŒÜŸLšL¨×:OÑ:OÔPˆÔÜ˜kÕ5Ó6ˆÜ ÐRaÔbˆŽr   c                óF   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[/# )r   rz   r?   r@   r    )r.   rB   r   r   )r   r   s   "€r   r   rï   )  s.   ø€ ÷ ñ ¡U§\¡\ð ¹3ð Ásð Ñ_cñ r   c                ó*  € V P                  V P                  V4      VVVR 7      pV^ ,          pVR,          pV P                  V4      pWa,           pV P                  V P	                  V4      4      pV P                  V4      pW,           p	V	3V,           pV# ))rz   r?   r@   r    rX   )rò   rñ   rí   rõ   rô   )
r:   rz   r?   r@   r    Úself_attention_outputsr¾   r²   Ú
mlp_outputÚlayer_outputs
   &&&&&     r   rb   ÚPvtLayer.forward)  sœ   € Ø!%§¡Ø×+Ñ+¨MÓ:ØØØ/ð	 "0ó "
Ðð 2°!Õ4ÐØ(¨Õ,ˆàŸ>™>Ð*:Ó;ÐØ(Õ8ˆà—X‘X˜d×/Ñ/°Ó>Ó?ˆ
à—^‘^ JÓ/ˆ
Ø$Õ1ˆà�/ GÕ+ˆàˆr   )rò   rí   rñ   rô   rõ   rd   r€   rm   s   @@r   rë   rë     s    ù‡ € ÷có c÷,÷ ò r   rë   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
PvtEncoderi@  c                ó    <€ V ^8„  d   QhRS[ /# ©r   r   r   )r   r   s   "€r   r   ÚPvtEncoder.__annotate__A  s   ø€ ÷ 0[ñ 0[™yñ 0[r   c                ór  <€ \         S	V `  4        Wn        \        P                  ! ^ VP
                  \        VP                  4      RR7      P                  4       p. p\        VP                  4       FÑ  pTP                  \        TV^ 8X  d   VP                  M*V P                  P                  ^V^,           ,          ,          VP                  V,          VP                  V,          V^ 8X  d   VP                   MVP"                  V^,
          ,          VP"                  V,          WAP                  ^,
          8H  R7      4       KÓ  	  \$        P&                  ! V4      V n        . p^ p\        VP                  4       Få  p. pV^ 8w  d!   WaP                  V^,
          ,          ,          p\        VP                  V,          4       Ft  pVP                  \+        VVP"                  V,          VP,                  V,          W&V,           ,          VP.                  V,          VP0                  V,          R7      4       Kv  	  VP                  \$        P&                  ! V4      4       Kç  	  \$        P&                  ! V4      V n        \$        P4                  ! VP"                  R,          VP6                  R7      V n        R# )r‹   Úcpu)rÜ   )r   r   r   r   r   r   r   )r   r   r…   rí   r†   rî   r$   NrG   )r'   r(   r   r.   ÚlinspaceÚdrop_path_rateÚsumÚdepthsÚtolistÚrangeÚnum_encoder_blocksÚappendr   r   Úpatch_sizesÚstridesr   Úhidden_sizesr   Ú
ModuleListÚpatch_embeddingsrë   r…   Úsequence_reduction_ratiosÚ
mlp_ratiosÚblockr4   r5   r6   )
r:   r   Údrop_path_decaysr>   ÚiÚblocksÚcurÚlayersÚjr;   s
   &&       €r   r(   ÚPvtEncoder.__init__A  s  ø€ Ü‰ÑÔØŒô !Ÿ>š>¨!¨V×-BÑ-BÄCÈÏÉÓDVÐ_dÔe×lÑlÓnÐð ˆ
ä�v×0Ñ0Ö1ˆAØ×ÑÜ"Ø!Ø45¸´F˜v×0Ò0ÀÇÁ×@VÑ@VÐ[\ÐabÐefÕafÕ[gÕ@hØ%×1Ñ1°!Õ4Ø!Ÿ>™>¨!Õ,Ø89¸Q¼ ×!4Ò!4ÀF×DWÑDWÐXYÐ\]ÕX]ÕD^Ø &× 3Ñ 3°AÕ 6Ø×#<Ñ#<¸qÕ#@Ñ@ôö
ñ 2ô !#§¢¨jÓ 9ˆÔð ˆØˆÜ�v×0Ñ0Ö1ˆAàˆFØ�AŒvØ—}‘} Q¨¥UÕ+Õ+�Ü˜6Ÿ=™=¨Õ+Ö,�Ø—‘ÜØ%Ø$*×$7Ñ$7¸Õ$:Ø,2×,FÑ,FÀqÕ,IØ"2¸µ7Õ";Ø28×2RÑ2RÐSTÕ2UØ"(×"3Ñ"3°AÕ"6ôö	ñ -ð �M‰Mœ"Ÿ-š-¨Ó/Ö0ñ! 2ô$ —]’] 6Ó*ˆŒ
ô Ÿ,š, v×':Ñ':¸2Õ'>ÀF×DYÑDYÔZˆŽr   c                ó†   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[R,          RS[R,          RS[S[,          /# ©r   rR   r    NÚoutput_hidden_statesÚreturn_dictrA   ©r.   ÚFloatTensorr   rS   r   )r   r   s   "€r   r   r  s  sT   ø€ ÷ #
ñ #
á×'Ñ'ð#
ñ   $�;ð#
ñ # T�kð	#
ñ
 ˜D•[ð#
ñ 
‘Õ	 ñ#
r   c                óè  € V'       d   RMR pV'       d   RMR pVP                   ^ ,          p\        V P                  4      pTp	\        \	        V P
                  V P                  4      4       Fœ  w  p
w  r¼V! V	4      w  r�pV FA  pV! W�Wâ4      pV^ ,          p	V'       d   VV^,          3,           pV'       g   K8  WY3,           pKC  	  W¨^,
          8w  g   Ki  V	P                  W}VR4      P                  ^ ^^^4      P                  4       p	Kž  	  V P                  V	4      p	V'       d
   WY3,           pV'       g1   \        ;QJ d    . R W•V3 4       F  NK  	  5# ! R W•V3 4       4      # \        V	VVR7      # )Nc              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irt   © )Ú.0Úvs   & r   Ú	<genexpr>Ú%PvtEncoder.forward.<locals>.<genexpr>‘  s   é € ÐmÑ$[˜qŸšÓ$[ùs   ‚Œ
©Úlast_hidden_staterz   Ú
attentionsr$  rG   )rY   Úlenr  Ú	enumerateÚzipr  rJ   rK   r©   r6   rS   r   )r:   rR   r    r  r  Úall_hidden_statesÚall_self_attentionsr_   Ú
num_blocksrz   ÚidxÚembedding_layerÚblock_layerr?   r@   r  Úlayer_outputss   &&&&&            r   rb   ÚPvtEncoder.forwards  sb  € ÷ #7™B¸DÐß$5™b¸4Ðà!×'Ñ'¨Õ*ˆ
Ü˜Ÿ™“_ˆ
Ø$ˆÜ3<¼SÀ×AVÑAVÐX\×XbÑXbÓ=cÖ3dÑ/ˆCÑ/�/á+:¸=Ó+IÑ(ˆM 5ã$�Ù % m¸UÓ V�Ø -¨aÕ 0�ß$Ø*=ÀÈqÕAQÐ@SÕ*SÐ'ß'Ñ'Ø(9Ð<LÕ(LÒ%ñ %ð  1•nÖ$Ø -× 5Ñ 5°jÈ%ÐQSÓ T× \Ñ \Ð]^Ð`aÐcdÐfgÓ h× sÑ sÓ u’ñ 4eð Ÿ™¨Ó6ˆßØ 1Ð4DÕ DÐßß”5Ñm ]ÐGZÑ$[Óm—5Ðm�5Ñm ]ÐGZÑ$[ÓmÓmÐmÜØ+Ø+Ø*ô
ð 	
r   )r  r   r6   r  )FFTr€   rm   s   @@r   rÿ   rÿ   @  s!   ù‡ € ÷0[ó 0[÷d#
÷ #
ò #
r   rÿ   c                   ó‚   a a€ ] tR tRt oRtRtRt. t]P                  ! 4       V3R lV 3R ll4       t
V3R ltRtVtV ;t# )	ÚPvtPreTrainedModeli™  ÚpvtrR   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# )r   ÚmodulerA   N)r   ÚModule)r   r   s   "€r   r   ÚPvtPreTrainedModel.__annotate__¢  s#   ø€ ÷ Hñ H¡B§I¡Ið H°$ñ Hr   c                ó8  <€ \         SV `  V4       V P                  P                  p\	        V\
        P                  \
        P                  34      '       dV   \        P                  ! VP                  RVR7       VP                  e#   \        P                  ! VP                  4       R# R# \	        V\        4      '       dY   \        P                  ! VP                  RVR7       VP                  e&   \        P                  ! VP                  RVR7       R# R# R# )zInitialize the weightsrÚ   )ÚmeanÚstdN)r'   Ú_init_weightsr   Úinitializer_ranger)   r   ru   r2   ÚinitÚtrunc_normal_Úweightr�   Úzeros_r   r0   r   )r:   r;  r@  r;   s   && €r   rA  Ú PvtPreTrainedModel._init_weights¡  sÄ   ø€ ô 	‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœrŸy™y¬"¯)©)Ð4×5Ò5Ü×Ò˜vŸ}™}°3¸CÕ@Ø�{‰{Ò&Ü—’˜FŸK™KÖ(ñ 'ä˜Ô 2×3Ò3Ü×Ò˜v×9Ñ9ÀÈÕMØ×ÑÒ+Ü×"Ò" 6×#3Ñ#3¸#À3×Gñ ,ñ 4r   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r  r   )r   r   s   "€r   r   r=  ™  s   ø‡ ‚ áÑò r   r$  )Úimage)re   rf   rg   rh   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesr.   Úno_gradrA  Ú__annotate_func__rj   rk   rl   rm   s   @@r   r8  r8  ™  sE   ù‡ € ð ÐØ$€OØ!ÐØÐà
‡]‚]ƒ_÷Hò Hó ðH÷ … r   r8  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚPvtModeli°  c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r   r   s   "€r   r   ÚPvtModel.__annotate__²  s   ø€ ÷ ñ ™yñ r   c                ór   <€ \         SV `  V4       Wn        \        V4      V n        V P                  4        R # rt   )r'   r(   r   rÿ   ÚencoderÚ	post_init©r:   r   r;   s   &&€r   r(   ÚPvtModel.__init__²  s/   ø€ Ü‰Ñ˜Ô ØŒô " &Ó)ˆŒð 	�‰Ör   c                ó†   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[R,          RS[R,          RS[S[,          /# r  r   )r   r   s   "€r   r   rS  ½  sT   ø€ ÷ 
ñ 
á×'Ñ'ð
ñ   $�;ð
ñ # T�kð	
ñ
 ˜D•[ð
ñ 
‘Õ	 ñ
r   c                ó\  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pV P	                  VVVVR7      pV^ ,          pV'       g   V3VR,          ,           # \        VVP                  VP                  R7      # )N©rR   r    r  r  rX   r)  )r   r    r  r  rU  r   rz   r+  )r:   rR   r    r  r  ÚkwargsÚencoder_outputsÚsequence_outputs   &&&&&,  r   rb   ÚPvtModel.forward¼  sµ   € ð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàŸ,™,Ø%Ø/Ø!5Ø#ð	 'ó 
ˆð *¨!Õ,ˆçØ#Ð%¨¸Õ(;Õ;Ð;äØ-Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r   )r   rU  )NNN©
re   rf   rg   rh   r(   r	   rb   rj   rk   rl   rm   s   @@r   rQ  rQ  °  s-   ù‡ € ÷ó ð ÷
ñ 
ó ÷
ð 
r   rQ  z¤
    Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    )Úcustom_introc                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚPvtForImageClassificationiÝ  c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   r   rA   Nr   )r   r   s   "€r   r   Ú&PvtForImageClassification.__annotate__ä  s   ø€ ÷ ñ ™yð ¨Tñ r   c                óB  <€ \         SV `  V4       VP                  V n        \        V4      V n        VP                  ^ 8”  d3   \
        P                  ! VP                  R,          VP                  4      M\
        P                  ! 4       V n	        V P                  4        R# )r‹   NrG   )r'   r(   Ú
num_labelsrQ  r9  r   ru   r  ró   Ú
classifierrV  rW  s   &&€r   r(   Ú"PvtForImageClassification.__init__ä  sy   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜FÓ#ˆŒð FL×EVÑEVÐYZÔEZŒB�IŠI�f×)Ñ)¨"Õ-¨v×/@Ñ/@ÔAÔ`b×`kÒ`kÓ`mð 	Œð
 	�‰Ör   c                ó¼   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )r   rR   NÚlabelsr    r  r  rA   )r.   rB   r   rS   r   )r   r   s   "€r   r   re  ó  sk   ø€ ÷ )
ñ )
á—l‘l TÕ)ð)
ñ —‘˜tÕ#ð)
ñ   $�;ð	)
ñ
 # T�kð)
ñ ˜D•[ð)
ñ 
Ñ&Õ	&ñ)
r   c                ó‚  € Ve   TMV P                   P                  pV P                  VVVVR7      pV^ ,          pV P                  VR,          4      p	Rp
Ve   V P	                  W)V P                   4      p
V'       g"   V	3VR,          ,           pV
e   V
3V,           # T# \        V
V	VP                  VP                  R7      # )ab  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr[  rX   )ÚlossÚlogitsrz   r+  )rW   r‹   rW   )r   r  r9  rh  Úloss_functionr   rz   r+  )r:   rR   rk  r    r  r  r\  r²   r^  rn  rm  r¹   s   &&&&&&,     r   rb   Ú!PvtForImageClassification.forwardò  sÇ   € ð  &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(Ø%Ø/Ø!5Ø#ð	 ó 
ˆð " !�*ˆà—‘ °Õ!9Ó:ˆàˆØÒØ×%Ñ% f°d·k±kÓBˆDçØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r   )rh  rg  r9  )NNNNr`  rm   s   @@r   rc  rc  Ý  s-   ù‡ € ÷ó ð ÷)
ñ )
ó ÷)
ð )
r   rc  )rc  rQ  r8  ))ri   r*   r¥   Úcollections.abcr   r.   Útorch.nn.functionalr   r§   rL   Ú r   rC  Úactivationsr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr	   r
   Úconfiguration_pvtr   Ú
get_loggerre   Úloggerr<  r   ro   r‚   rµ   rÁ   rÒ   rë   rÿ   r8  rQ  rc  Ú__all__r$  r   r   Ú<module>r|     s7  ðñ  ã Û Ý $ã ß Ð Ý å &Ý !ß FÝ -ß ,Ý (ð 
×	Ò	˜HÓ	%€ôA)˜Ÿ™ô A)ôH	�B—I‘Iô 	ôO §	¡	ô Oôd�2—9‘9ô ô.ˆR�Y‰Yô ô8%�"—)‘)ô %ô0+ˆr�y‰yô +ô\V
�—‘ô V
ðr ôH˜ó Hó ðHð, ô)
Ð!ó )
ó ð)
ñX ðôô9
Ð 2ó 9
óð9
òx J‚r   