+
j * R t ^ RIt^ RIHt ^ RIHt ^ RIt^ RI t ^ RI H
t
^RI H t
^RIHt ^RIHt ^RIHt ^R IHtHt ^R
IHt ^R IHt ^R IH t H t H t H t H t H!t! ^R
I"H#t# ^RI$H%t% ^RI&H't'H(t(H)t) ] PT ! ]+4 t,R R lt-R R lt.R R lt/RBR R llt0RCR lt1R t2 ! R R]
Pf 4 t4 ! R R ]
Pf 4 t5 ! R R ]
Pf 4 t6] ] ! R R!] 4 4 4 t7 ! R" R#]
Pf 4 t8 ! R$ R%]
Pf 4 t9 ! R& R']
Pf 4 t: ! R( R)]
Pf 4 t; ! R* R+]
Pf 4 t< ! R, R-]<4 t= ! R. R/]
Pf 4 t> ! R0 R1]4 t?] ! R2 R3]4 4 t@ ! R4 R5]
Pf 4 tA ! R6 R7]
Pf 4 tB ! R8 R9]@4 tC ! R: R;]@4 tD ! R< R=]
Pf 4 tE ! R> R?]@4 tF] ! R@ RA]@4 4 tG. RDOtHR# )EzPyTorch GroupViT model.N) dataclass)Any)nn)initialization)ACT2FN)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPooling)PreTrainedModel)Unpack) ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging torch_int)merge_with_config_defaults)capture_outputs)GroupViTConfigGroupViTTextConfigGroupViTVisionConfigc X V ^8 d QhR\ P R\ P /# ) logitsreturntorchTensor)formats "o/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/groupvit/modeling_groupvit.py __annotate__r! * s& ` `U\\ `ell ` c \ P P V \ P ! \ V 4 V P R 7 4 # )device)r
functional
cross_entropyr arangelenr% )r s &r contrastive_lossr* * s/
== & &vu||CKPVP]P]/^ __r" c X V ^8 d QhR\ P R\ P /# )r
similarityr r )r s "r r! r! / s" - -ELL -U\\ -r" c ` \ V 4 p\ V P 4 pW, R , # )g @)r* T)r, caption_loss
image_losss & r image_text_contrastive_lossr1 / s( #J/L!*,,/J % ,,r" c D V ^8 d QhR\ P R\ /# )r r dimr r int)r s "r r! r! 5 s C r" c V P V4 pVP VR R7 ^, p\ P ! V \ P R7 P WR4 pWBP
4 ,
V, pV# )Tkeepdim
memory_format ?)softmaxmaxr
zeros_likelegacy_contiguous_formatscatter_detach)r r3 y_softindexy_hardrets && r hard_softmaxrF 5 sf
^^C
F JJsDJ )! ,E
fE4R4R
S
\
\]`il
mF
==?
"V
+C Jr" c
| V ^8 d QhR\ P R\ R\ R\ R\ P /# )r r tauhardr3 r )r r floatboolr5 )r s "r r! r! ? s; 5<< e t RU _d_k_k r" c
r \ P P P \ P ! RR V P
V P R7 \ P ! RRV P
V P R7 4 pVP V P 4 pW, V, pVP V4 pV' dp VP VRR7 ^, p\ P ! V \ P R7 P W7R4 pWP 4 ,
V, p V # Tp V # ) )r% dtyper; Tr7 r9 )r
distributionsgumbelGumbelfullr% rN sampleshaper< r= r> r? r@ rA )
r rH rI r3 gumbel_distgumbelsrB rC rD rE s
&&&& r gumbel_softmaxrX ? s %%,,33
2s6== E
2s6== EK .G 3&G
__S
!F
3
-a0!!&8V8VW``admpq}}&/ J Jr" c W, V P ^, , R, pW8 dA \ \ P ! W$, 4 4 pV P ^, V, pM?\ \ P ! W, 4 4 pV P ^, V, pV P ^ , pV P ^, pV P WxWe4 p \
P P WV3RVR7 p V # )a
Args:
attentions (`torch.Tensor`): attention map of shape [batch_size, groups, feat_height*feat_width]
height (`int`): height of the output attention map
width (`int`): width of the output attention map
align_corners (`bool`, *optional*): the `align_corner` argument for `nn.functional.interpolate`.
Returns:
`torch.Tensor`: resized attention map of shape [batch_size, groups, height, width]
?bilinearsizemode
align_corners)rU r5 nproundreshaper r& interpolate)
attentionsheightwidthr_ scale
feat_width feat_height
batch_sizegroupss &&&& r resize_attention_maprl U s ^z / / 2
2s :E
~%-01
&&q)Z7 "((6>23 %%a(K7
!!!$J
a
F##J PJ **% z + J r" c
~ . p\ P ! 4 ;_uu_ 4 RpV Fp pVP ^ ^^4 P 4 pVf TpMW4, p\ VP ^ ^^4 P 4 .VO5! pVP V4 Kr RRR4 VR, pV# + ' g i L; i)a
Args:
attentions (`tuple(torch.FloatTensor)`: tuple of attention maps returned by `GroupViTVisionTransformer`
hw_shape (`tuple(int)`): height and width of the output attention map
Returns:
`torch.Tensor`: the attention map of shape [batch_size, groups, height, width]
N)r no_gradpermute
contiguousrl append)rd hw_shape attn_mapsprev_attn_masks
attn_masks cur_attn_mapfinal_groupings && r get_grouping_from_attentionsry s s I $J#++Aq!4??AJ &","1">/0G0G1a0P0[0[0]i`hiL \ * %
r]N !
s A9B,, B< c D a a ] tR t^t oV3R lV 3R lltR tRtVtV ;t# )GroupViTCrossAttentionLayerc < V ^8 d QhRS[ /# r configr )r
__classdict__s "r r! (GroupViTCrossAttentionLayer.__annotate__ s U U 3 Ur" c ( < \ SV ` 4 \ V4 V n \ P
! VP VP R 7 V n \ V4 V n
\ P
! VP VP R 7 V n R# epsN) super__init__GroupViTAttentionattnr LayerNorm hidden_sizelayer_norm_epsnorm2 GroupViTMLPmlp norm_postselfr~ __class__s &&r r $GroupViTCrossAttentionLayer.__init__ sb
%f- \\&"4"4&:O:OP
v& f&8&8f>S>STr" c TpW0P WR 7 ^ , , pW0P V P V4 4 , pV P V4 pV# ))encoder_hidden_statesr r r r )r querykeyxs &&& r forward#GroupViTCrossAttentionLayer.forward sK
% ;A> >
A ' ' NN1 r" r __name__
__module__ __qualname____firstlineno__r r __static_attributes____classdictcell__
__classcell__r r s @@r r{ r{ s U U r" r{ c N a a ] tR t^t oV3R lV 3R lltRR ltR tRtVtV ;t # )GroupViTAssignAttentionc < V ^8 d QhRS[ /# r} r )r r s "r r! $GroupViTAssignAttention.__annotate__ s , , 3 ,r" c < \ SV ` 4 VP R, V n \ P
! VP VP 4 V n \ P
! VP VP 4 V n \ P
! VP VP 4 V n \ P
! VP VP 4 V n VP V n
R# )rZ N ) r r r rg r Linearq_projk_projv_projproj
assign_epsr s &&r r GroupViTAssignAttention.__init__ s
''-
ii 2 2F4F4FG ii 2 2F4F4FG ii 2 2F4F4FG IIf 0 0&2D2DE ++r" c V' d# V P ' d \ VRVR7 pV# V' d \ VRR7 pV# \ P P VRR7 pV# )r )r3 rI r3 )trainingrX rF r r& r< )r r rQ rI s &&&&r get_attn GroupViTAssignAttention.get_attn sZ dmmm!$BT:D #Db1 }},,Tr,: r" c TpV P V4 pV P V4 pV P V4 pWP RR4 , V P , pV P V4 pV P VRRR7 pWUP
RRR7 V P , , pWS, pV P V4 pWv3# )r F)rQ rI Tr3 r8 r rn ) r r r transposerg r sumr r )r r r valueraw_attnr soft_attnouts &&& r r GroupViTAssignAttention.forward s E" kk# E" MM"b 11TZZ?}}X&MM(5uME xxBx5GHliin~ r" )r r r r rg r )TT)
r r r r r r r r r r r s @@r r r s , , r" r c J a a ] tR t^t oV3R lV 3R lltR tR tRtVtV ;t # )GroupViTTokenAssignc < V ^8 d QhRS[ /# r} r )r r s "r r! GroupViTTokenAssign.__annotate__ s f f 3 fr" c x < \ SV ` 4 W0n \ P ! VP
VP R 7 V n \ VP \ P P 4 ' d
VP MVP VP 3pV Uu. uF p\ WQP
, 4 NK upw rg\ WWc4 V n \ P ! VP
VP R 7 V n \ P ! VP
VP R 7 V n \% V4 V n \) V4 V n \ P ! VP
VP R 7 V n \/ WP
WqP
4 V n R# u upi r )r r num_output_groupr r r r norm_tokens
isinstanceassign_mlp_ratio collectionsabcIterabler5 GroupViTMixerMLP mlp_internorm_post_tokensnorm_xr{ pre_assign_attnr assign
norm_new_xr mlp_channels) r r~ num_group_tokenr r r
tokens_dim channels_dimr s &&&& r r GroupViTTokenAssign.__init__ sD
0 <<(:(:@U@UV &11;??3K3KLL
# #))6+B+BC
JZ#ZIYAC,>,>(>$?IY#Z
)&:` " V-?-?VEZEZ [ ll6#5#56;P;PQ :6 B -f5 ,,v'9'9v?T?TU '0B0BLRdRd e $[s #$F7c J V P V4 pV P V4 pV# )z
Args:
group_tokens (torch.Tensor): group tokens, [batch_size, num_group_tokens, channels]
Returns:
projected_group_tokens (torch.Tensor): [batch_size, num_output_groups, channels]
)r r )r group_tokensprojected_group_tokenss && r project_group_token'GroupViTTokenAssign.project_group_token s+ "& != !%!6!67M!N %%r" c V P V4 pV P V4 pV P V4 pV P W14 pV P W14 w rEWC,
pW@P V P
V4 4 , pWE3# )z
Args:
image_tokens (`torch.Tensor`): image tokens, of shape [batch_size, input_length, channels]
group_tokens (`torch.Tensor`): group tokens, [batch_size, num_group_tokens, channels]
)r r r r r r r )r image_tokensr r new_image_tokens attentions &&& r r GroupViTTokenAssign.forward s '' 5 {{<0 !%!9!9,!G !%!5!56L![ &*kk2H&W#2+.?.?P`@a.bb **r" ) r r r r r r r r r )
r r r r r r r r r r r s @@r r r s f f* &+ +r" r c b a ] tR tRt o RtRtRtRtRtRt Rt
Rt Rt V 3R lR lt
V 3R ltRtV tR# )GroupViTModelOutputi a
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
Contrastive loss for image-text similarity.
logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
similarity scores.
logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
similarity scores.
segmentation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels, logits_height, logits_width)`):
Classification scores for each pixel.
The logits returned do not necessarily have the same size as the `pixel_values` passed as inputs. This is
to avoid doing two interpolations and lose some quality when a user needs to resize the logits to the
original image size as post-processing. You should always check your logits shape and resize as needed.
text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
The text embeddings obtained by applying the projection layer to the pooled output of
[`GroupViTTextModel`].
image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
The image embeddings obtained by applying the projection layer to the pooled output of
[`GroupViTVisionModel`].
text_model_output (`BaseModelOutputWithPooling`):
The output of the [`GroupViTTextModel`].
vision_model_output (`BaseModelOutputWithPooling`):
The output of the [`GroupViTVisionModel`].
Nc 0 < V ^8 d QhRS[ S[, /# r r )tupler )r r s "r r! GroupViTModelOutput.__annotate__2 s
%*
r" c a \ ;QJ d% . V 3R lS P 4 4 F NK 5# ! V 3R lS P 4 4 4 # )c 3 x <" T F/ pVR9 d
SV, M\ SV4 P 4 x K1 R# 5i)text_model_outputN)r vision_model_output)getattrto_tuple).0kr s & r /GroupViTModelOutput.to_tuple..3 s<
LLDGRYZ^`aRbRkRkRm m s 7:)r keysr s fr r GroupViTModelOutput.to_tuple2 sA u
YY[
u
u
YY[
r" c F < V ^8 d Qh/ S[ P R, ;R& S[ P R, ;R& S[ P R, ;R& S[ P R, ;R& S[ P R, ;R& S[ P R, ;R& S[;R& S[;R & # )
r Nlosslogits_per_imagelogits_per_textsegmentation_logits text_embeds image_embedsr r )r FloatTensorr
)r r s "r r! r s D
d
")E F ''$.5G H &&-4I J **T18K L ""T)0M N ##d*1O P 28Q R 4:S r" rO )r r r r __doc__r r r r r r r r r __annotate_func__r r )r s @r r r sJ > &*D1504O48,0K-1L486:
W r" r c \ a a ] tR tRt oRtRV3R lV 3R llltR V3R lR lltRtVtV ;t # )
GroupViTPatchEmbeddingsi9 z
Image to Patch Embedding.
c < V ^8 d QhRS[ S[S[ , , S[S[ S[ 3, , RS[ S[S[ S[ 3, , RS[ RS[ /# )r
image_size
patch_size num_channels embed_dim)r5 listr )r r s "r r! $GroupViTPatchEmbeddings.__annotate__> s[ h h$s)OeCHo5h %S/)h h
hr" c < \ SV ` 4 \ V\ P P
4 ' d TMW3p\ V\ P P
4 ' d TMW"3pV^, V^, , V^ , V^ , , , pWn W n WPn \ P ! W4W"R7 V n R# ) ) kernel_sizestrideN) r r r r r r r r num_patchesr Conv2d
projection)r r r r r r r s &&&&& r r GroupViTPatchEmbeddings.__init__> s #-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
1
5*Q-:VW=:XY $$& ))Lgr" c T < V ^8 d QhRS[ P RS[RS[ P /# r pixel_valuesinterpolate_pos_encodingr r r rK )r r s "r r! r O s- ELL D ]b]i]i r" c d VP w r4rVV' gi WPP ^ , 8w g W`P ^, 8w d; \ RV RV RV P ^ , RV P ^, R2 4 hV P V4 P ^4 P ^^4 pV# ) zInput image size (*z) doesn't match model ().)rU r
ValueErrorr flattenr )r r r rj r re rf r s &&& r r GroupViTPatchEmbeddings.forwardO s 2>2D2D/
&' ++u8J/J (% 9 +,Adooa.@-AE
OOL ) 1 1! 4 > >q! Dr" )r r r r ) i F
r r r r r r r r r r r s @@r r r 9 s% h h" r" r c f a a ] tR tRt oV3R lV 3R lltV3R lR ltR V3R lR lltRtVtV ;t # )
GroupViTVisionEmbeddingsi[ c < V ^8 d QhRS[ /# r} r )r r s "r r! %GroupViTVisionEmbeddings.__annotate__\ s 3 r" c < \ SV ` 4 \ VP VP VP
VP R 7 V n V P P p\ P ! \ P ! ^W!P 4 4 V n
\ P ! VP 4 V n \ P ! VP VP" R7 V n VP V n Wn R# ))r r r r r N)r r r r r r r patch_embeddingsr r Parameterr zerosposition_embeddingsDropoutdropoutr r layernormr~ )r r~ r r s && r r !GroupViTVisionEmbeddings.__init__\ s
7 (( (( ,, (( !
++77 #%<< A{L^L^0_#` zz&..1 f&8&8f>S>ST ++ r" c Z < V ^8 d QhRS[ P RS[RS[RS[ P /# )r
embeddingsre rf r r4 )r r s "r r! r& l s5 $ $ 5<< $ $ UX $ ]b]i]i $ r" c h VP ^, pV P P ^, p\ P P 4 ' g WE8X d W#8X d
V P # V P pVP R, pW P
, pW0P
, p \
VR, 4 p
VP ^WV4 pVP ^ ^^^4 p\ P P VW3RRR7 pVP ^ ^^^4 P ^RV4 pV# )a
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing and no class embeddings.
Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
rZ bicubicFr\ rn )
rU r+ r jit
is_tracingr r rb rp r r& rc view) r r1 re rf r
num_positionspatch_pos_embedr3
new_height new_widthsqrt_num_positionss &&&& r r 1GroupViTVisionEmbeddings.interpolate_pos_encodingl s !&&q) 0066q9
yy##%%+*F6?++ + 22 r".
__, &}c'9 :)11!5G]`a)11!Q1= --33 ( 4
*11!Q1=BB1b#N r" c T < V ^8 d QhRS[ P RS[RS[ P /# r r )r r s "r r! r& s- ELL D ]b]i]i r" c VP w r4rVV P WR 7 pV P V4 pVP 4 w r8p V' d WpP WuV4 , pMWpP
, pV P
V4 pV# ))r )rU r( r. r] r r+ r- )
r r r rj r re rf r1 seq_len_s
&&& r r GroupViTVisionEmbeddings.forward s 2>2D2D/
&**<*k
^^J/
!+ !2
Q $#&C&CJX]&^^J#&>&>>J\\*-
r" )r~ r- r. r( r r+ r! )
r r r r r r r r r r r s @@r r$ r$ [ s) $ $ L r" r$ c T a a ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )GroupViTTextEmbeddingsi c < V ^8 d QhRS[ /# r} r )r r s "r r! #GroupViTTextEmbeddings.__annotate__ s v v 1 vr" c h < \ SV ` 4 VP p\ P ! VP
V4 V n \ P ! VP V4 V n \ P ! \ P ! VP 4 P R4 RR7 V n
R# )r F)
persistentNr rn )r r r r Embedding
vocab_sizetoken_embeddingmax_position_embeddingsposition_embeddingBufferr r( expand position_idsr r~ r r s && r r GroupViTTextEmbeddings.__init__ s|
&& !||F,=,=y I "$,,v/M/My"Y IIell63Q3Q&R&Y&YZa&bot ur" c < V ^8 d QhRS[ P R, RS[ P R, RS[ P R, RS[ P /# )r input_idsNrQ
inputs_embedsr )r
LongTensorr r )r r s "r r! rF sX ##d* &&- ((4/
r" c h Ve VP R, MVP R, pV P P P ^ , pWE8 d \ RV RV 24 hVf V P RR V13, pVf V P V4 pV P V4 pW6, pV# )NzRSequence length must be less than max_position_embeddings (got `sequence length`: z and max_position_embeddings: NNNrn r )rU rN weightr rQ rL )r rU rQ rV
seq_lengthmax_position_embeddingr+ r1 s &&&& r r GroupViTTextEmbeddings.forward s -6,AY__R(}GZGZ[]G^
!%!8!8!?!?!E!Ea!H . d , <=S> )T^^ ; MdnnM] ^NN#2'
]]D(
// ii? ii? ii? $..$..A
r" c @ < V ^8 d QhRS[ P RS[RS[/# )r tensorr? bszr4 )r r s "r r! GroupViTAttention.__annotate__] s+ e eU\\ eC ec er" c VP W2V P V P 4 P ^^4 P 4 # r )r6 r r r rq )r r r? r s &&&&r _shapeGroupViTAttention._shape] s5 {{3 GQQRSUVWbbddr" c < V ^8 d QhRS[ P RS[ P R, RS[ P R, RS[S[ P S[ P R, 3, /# )r r} r Nr r )r r r r )r r s "r r! r ` sh D2 D2 ||D2 t+D2 % 0 04 7 D2
u||U\\D 00 1
D2r" c X VP 4 w rVpVRJpV P V4 V P , p V' dF V P V P V4 RV4 p
V P V P V4 RV4 p MDV P V P V4 RV4 p
V P V P V4 RV4 p WPP , RV P 3p V P WV4 P ! V ! p V
P ! V ! p
V P ! V ! p V
P ^4 p
\ P ! WP ^^4 4 pVP 4 WPP , Wm38w d2 \ RWPP , Wm3 RVP 4 24 hVe VP 4 V^Wm38w d# \ RV^Wm3 RVP 4 24 hVP WPP Wm4 V, pVP WPP , Wm4 p\ P P VRR7 pVP WPP Wm4 pVP WPP , Wm4 p\ P P! WP V P" R7 p\ P ! VV 4 pVP 4 WPP , W`P 38w d6 \ RWPP W`P 3 RVP 4 24 hVP WPP W`P 4 pVP ^^4 pVP% WVV4 pV P' V4 pVV3# ) z#Input shape: Batch x Time x ChannelNz$Attention weights should be of size z , but is z!Attention mask should be of size r )pr z `attn_output` should be of size rn )r] r rg r r r r r r6 r bmmr r r r& r< r- r rb r )r r} r r kwargsr tgt_lenr is_cross_attention query_states
key_states value_states
proj_shapesrc_len attn_weightsattn_weights_reshaped
attn_probs attn_outputs &&&&, r r GroupViTAttention.forward` s= #0"4"4"6 i 2$ > {{=1DJJ> T[[1F%GSQJ ;;t{{3H'I2sSL T[[ %?SIJ ;;t{{='A2sKLNN*B
>
{{<#>CCZP __j1
#((*5 //!$ yy /C/CAq/IJ 3 #7"J J 6nn8Lg7_6` a %%'(*
% ""$a(BB 7a8R7SS\]k]p]p]r\st (,,S..'SVddL',,S>>-A7TL}},,\r,B !- 1 1#~~w X ,11#2FY ]]**<<>
'
)
!0
(( 7
/
0
r" )r r r r r r r s @@r rj rj s S S r" rj c ~ a a ] tR tRt oRtR tRtR]R]/t ]
P ! 4 V 3R l4 t V3R lt
RtVtV ;t# )
GroupViTPreTrainedModeli groupvitTr} rd c < \ SV ` V4 V P P p\ V\
4 ' d \ P ! VP P RVR, R7 \ P ! VP P RVR, R7 \ P ! VP \ P ! VP P R, 4 P! R4 4 R# \ V\" 4 ' Ed& V P P pVP$ R, ^VP P& , R, , V, pVP$ R, V, p\ P ! VP( P VR7 \ P ! VP* P VR7 \ P ! VP, P VR7 \ P ! VP. P VR7 R# \ V\0 4 ' d V P P pVP P2 R, ^VP P& , R, , V, p^VP P2 , R, V, p\ P ! VP4 P VR7 \ P ! VP6 P VR7 R# R# ) zInitialize the weightsrM g{Gz?)meanstd)r Nrn rI r ) r
_init_weightsr~ initializer_factorr rC initnormal_rL rZ rN copy_rQ r r( rU rP r r num_hidden_layersr r r r r r r r )r modulefactor in_proj_std out_proj_stdfc_stdr s && r r %GroupViTPreTrainedModel._init_weights s f%// f 4 5 5 LL//66Sftm T LL2299RV W JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h
1
2
2[[33F!++T1q6==;Z;Z7Z_c6cdgmmK " , ,d 2f%+%=%=a%@LMPQE)A)A!a%)HWX 3
',#
s A-Cc < V ^8 d QhRS[ P RS[R, RS[R, RS[R, RS[S[, /# )r r} output_hidden_statesNr return_dictr )r r rK r r )r r s "r r! r sR %
%
||%
#Tk%
$; %
D[ %
%
r" c l Ve TMV P P pVe TMV P P pVe TMV P P pV' d RMR pV' d RMR pR p\ V P
4 FX w rV' d
WQ3, pV ! WV4 p
V
^ , pV
^, pV' g K; V
^, f KH Wj^, 3, pKZ V' d
WQ3, pV' g1 \ ;QJ d . R WV3 4 F NK 5# ! R WV3 4 4 # \ WVR7 # )Nc 3 0 " T F qf K Vx K R # 5irq rO )r vs & r r 0GroupViTVisionEncoder.forward.." s g$Uq$Us
)last_hidden_stater} rd rO )r~ r r r enumerater r r ) r r} r r r all_hidden_states
all_groupingsr r stage
layer_outputss &&&&& r r GroupViTVisionEncoder.forward s" 2C1N -TXT_T_TqTq$8$D $++JjJj &1%>> from transformers import CLIPTokenizer, GroupViTTextModel
>>> tokenizer = CLIPTokenizer.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> model = GroupViTTextModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
>>> outputs = model(**inputs)
>>> last_hidden_state = outputs.last_hidden_state
>>> pooled_output = outputs.pooler_output # pooled (EOS token) states
```rU r rQ rO r# )r rU r rQ r s &&&&,r r GroupViTTextModel.forward s9 .
)
&
r" c & < V ^8 d Qh/ S[ ;R& # r} rE )r r s "r r! r! s r" r1 )r r^ )r r r r r r r) r, r r r r r r r s @@r r r sD : :;
# r" r c ^ a a ] tR tRt oV3R lV 3R llt]RV3R lR ll4 tRtVtV ;t # )GroupViTVisionTransformeri c < V ^8 d QhRS[ /# r} r )r r s "r r! &GroupViTVisionTransformer.__annotate__ s L L 3 Lr" c < \ SV ` 4 Wn VP p\ V4 V n \
V4 V n \ P ! W!P R 7 V n R# r ) r r r~ r r$ r1 r r r r r r. rR s && r r "GroupViTVisionTransformer.__init__ sL
&& 26:,V4 i5J5JKr" c < V ^8 d QhRS[ P R, RS[R, RS[R, RS[R, RS[S[, /# )r r Nr r r r r r rK r r
)r r s "r r! r7 sZ '
'
''$.'
#Tk'
$; '
D[ '
+ +
'
r" c Ve TMV P P pVe TMV P P pVe TMV P P pVf \ R4 hV P V4 pV P
VVVVR7 pV^ , pV P V4 pVP ^R7 pV' g Wx3VR, , # \ VVVP VP R7 # )Nz You have to specify pixel_values)r} r r r r :r NN)r r r} rd ) r~ r r r r r1 r r. r r
r} rd ) r r r r r r} r r r s &&&&& r r !GroupViTVisionTransformer.forward s 2C1N -TXT_T_TqTq$8$D $++JjJj &1%>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, GroupViTVisionModel
>>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> model = GroupViTVisionModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
... image = Image.open(BytesIO(response.read()))
>>> inputs = processor(images=image, return_tensors="pt")
>>> outputs = model(**inputs)
>>> last_hidden_state = outputs.last_hidden_state
>>> pooled_output = outputs.pooler_output # pooled CLS states
```r r r r rD )r r r r r r s &&&&&,r r GroupViTVisionModel.forward s( > % /!5# !
r" c & < V ^8 d Qh/ S[ ;R& # r} r )r r s "r r! rB s r" rL )r r> )r r r r main_input_namer r r r) r r r r r r r s @@r r@ r@ sJ $O! = = #
#
#
! r" r@ c a a ] tR tRt oV3R lV 3R llt]]R V3R lR ll4 4 t]]V3R lR l4 4 t]]R
V3R lR ll4 4 t V3R
lt
R t Vt V ;t
# )
GroupViTModeli? c < V ^8 d QhRS[ /# r} r )r r s "r r! GroupViTModel.__annotate__C s ) )~ )r" c h < \ SV ` V4 \ VP \ 4 ' g# \ R \
VP 4 R24 h\ VP \ 4 ' g# \ R\
VP 4 R24 hVP pVP pVP V n VP V n
VP V n VP V n
\ V4 V n \! V4 V n \$ P& ! \$ P( ! V P V P RR7 \$ P* ! V P 4 \$ P, ! RR7 \$ P( ! V P V P RR7 4 V n \$ P& ! \$ P( ! V P V P RR7 \$ P* ! V P 4 \$ P, ! RR7 \$ P( ! V P V P RR7 4 V n \$ P2 ! \4 P6 ! V P8 P: 4 4 V n V P? 4 R# )zOconfig.text_config is expected to be of type GroupViTTextConfig but is of type .zSconfig.vision_config is expected to be of type GroupViTVisionConfig but is of type T)bias)inplaceN) r r r text_configr TypeErrortype
vision_configr projection_dimprojection_intermediate_dimr text_embed_dimvision_embed_dimr r# r5 rD r rm r BatchNorm1dReLUvisual_projectiontext_projectionr) r r r~ logit_scale_init_value logit_scaler )r r~ rX r[ r s && r r GroupViTModel.__init__C s
&,,.@AA++,-Q0
&..0DEE--./q2
(( ,,
$ 3 3+1+M+M( ) 5 5 - 9 9 1+> 5m D!# IId++T-M-MTX Y NN4;; < GGD ! IId668K8KRV W "
"}} IId))4+K+KRV W NN4;; < GGD ! IId668K8KRV W
<< T[[5W5W(XY
r" c < V ^8 d QhRS[ P RS[ P R, RS[ P R, RS[S[, RS[S[, /# )r rU r NrQ r r r/ )r r s "r r! rS p s_ << t+ llT)
+,
+ +
r" c ~ V P ! RRVRVRVRR/VB pVP pV P V4 Vn V# )a
Examples:
```python
>>> import torch
>>> from transformers import CLIPTokenizer, GroupViTModel
>>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> tokenizer = CLIPTokenizer.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
>>> with torch.inference_mode():
... text_features = model.get_text_features(**inputs)
```rU r rQ r TrO )r# r rc )r rU r rQ r text_outputsr s &&&&, r get_text_features GroupViTModel.get_text_featuresn sf . 48?? 4
4
)4
&4
4
4
%22
%)%9%9-%H "r" c ` < V ^8 d QhRS[ P RS[S[, RS[S[, /# )r r r r r/ )r r s "r r! rS s9 ll +,
+ + r" c p V P ! V3RR/VB pV P VP 4 Vn V# )a"
Examples:
```python
>>> import torch
>>> from transformers import AutoProcessor, GroupViTModel
>>> from transformers.image_utils import load_image
>>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = load_image(url)
>>> inputs = processor(images=image, return_tensors="pt")
>>> with torch.inference_mode():
... image_features = model.get_image_features(**inputs)
```r T)rD rb r )r r r vision_outputss &&, r get_image_features GroupViTModel.get_image_features s? 4 6:5F5F|5pae5pio5p'+'='=n>Z>Z'[$ r" c 6 < V ^8 d QhRS[ P R, RS[ P R, RS[ P R, RS[ P R, RS[R, RS[R, RS[R, R S[R, R
S[S[, R S[S[, /
# ) r rU Nr r rQ return_lossr r output_segmentationr r ) r rW r r rK r r r r )r r s "r r! rS s |
|
##d*|
''$.|
t+ |
&&- |
D[
|
$;|
#Tk|
"D[|
+,|
$ $|
r" c < Ve TMV P P pVe TMV P P pV' d RpVe TMV P P pV P VVVRR7 p
V P
! R
RVRVRV/V B p V
P p V P V 4 p V P p
V P V
4 p
WP R RR7 , p WP R RR7 , p
V P P 4 p\ P ! WP 4 4 V, pVP 4 pRpV' Ed V
P pV P VP! R VP" R , 4 4 pV
P$ p\' VVP" R, 4 pVVP R RR7 , p\ P ! VV
P 4 4 V, pVP! V P" ^ , R V
P" ^ , 4 P) ^ ^^4 pVP! VP" ^ , VP" ^, R 4 p\ P ! VV4 V, pVP! VP" ^ , VP" ^, VP" ^, VP" ^, 4 pRpV' d \+ V4 p\- VVVVV
V V V
R 7 # ) a
return_loss (`bool`, *optional*):
Whether or not to return the contrastive loss.
output_segmentation (`bool`, *optional*):
Whether or not to return the segmentation logits.
Examples:
```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, GroupViTModel
>>> model = GroupViTModel.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> processor = AutoProcessor.from_pretrained("nvidia/groupvit-gcc-yfcc")
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
... image = Image.open(BytesIO(response.read()))
>>> inputs = processor(
... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
... )
>>> outputs = model(**inputs)
>>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
>>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
```NTrK rU r rQ r :r NN)r r r r r r r r rO rn )r~ r rs r rD r# r rb rc normre expr matmultr rb rU rd ry rp r1 r )r rU r r rQ rr r r rs r rn ri r r re r r
seg_logitsimage_group_embedsrd groupinglogits_per_image_groupflatten_groupingr s &&&&&&&&&, r r GroupViTModel.forward s X 2C1N -TXT_T_TqTq#6#B HgHg $ $8$D $++JjJj
**% /!5 +
48?? 4
4
)4
&4
4
&33 --l; "00 **;7 $&7&7B&7&MM !$4$4T$4$JJ &&**, ,,{NN4DE S*,,.
"0!A!A !%!7!78J8R8RSUWiWoWoprWs8t!u '22J3J @R@RSU@VWH "46H6M6MRT^b6M6c!c %*\\2Dkmmo%VYd%d "%;%C%C ""1%r;+<+>RSCTVX Y &<>NOR]]J#++ #Z%5%5a%8(..:KX^^\]M^J .?D" - + *#% * .
r" c & < V ^8 d Qh/ S[ ;R& # r} r )r r s "r r! rS ? r r" ) re r\ r] r^ r# rc r_ rD rb r )NNNNNNNN)r r r r r r r rj ro r r r r r r s @@r rQ rQ ? sz ) )V B : |
|
|
g r" rQ )r Frn r! )rQ r r r@ )Ir collections.abcr dataclassesr typingr numpyr` r r r r activationsr
masking_utilsr modeling_layersr modeling_outputsr r
modeling_utilsr processing_utilsr utilsr
r r r r r
utils.genericr utils.output_capturingr configuration_groupvitr r r
get_loggerr loggerr* r1 rF rX rl ry r& r{ r r r r r$ rC r` r r r rj r r r r r r5 r@ rQ __all__rO r" r r s ! & ! / 9 K - & j j 7 5 \ \
H %
`
- ,<:
"))
- bii - `4+")) 4+n
-
+ -
-
` bii DGryy GV#RYY #LZBII Zz ")) 0!{ ! ]2 ]2B 5 B !=o != !=H7
BII 7
t-
")) -
`J
5 J
Z,
/ ,
^2
2
j3
1 3
l n
+ n
n
b cr"