+ j]4^RIHtHt^RIt^RIHt^RIHt^RIH t ^RI H t ^RI H t ^RIHtHtHt^R IHtHt^R IHt^R IHtHtHt^R IHtHt^R IHt^RI H!t!!RR]PD4t#!RR]PD4t$R%RRllt%!RR]PD4t&!RR]PD4t'!RR] 4t(!RR]PD4t)]!RR]44t*]!RR ]*44t+]!R!R"7!R#R$]*44t,.R&Ot-R#)')CallableIterableN)nn)initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring torch_int)can_return_tuplemerge_with_config_defaults)capture_outputs VivitConfigcTaa]tRt^'toRtV3RlV3RlltV3RlRltRtVtV;t #)VivitTubeletEmbeddingsae This class turns `pixel_values` of shape `(batch_size, num_frames, num_channels, height, width)` into the initial `hidden_states` (tubelet embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a Transformer encoder. The seq_length equals (num_frames // tubelet_size[0]) * (height // tubelet_size[1]) * (width // tubelet_size[2]). c <V^8dQhRS[/#configr)format __classdict__s"i/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/vivit/modeling_vivit.py __annotate__#VivitTubeletEmbeddings.__annotate__0s  { c<\SV`4VPpVPp\ V\ 4'dTMW33pVP V^,,V^,V^,,,V^,V^,,,VnW0n\P!VPVPW"R7Vn R#)r) kernel_sizestrideN) super__init__ tubelet_size image_size isinstancer num_frames num_patchesrConv3d num_channels hidden_size projection)selfrr)r* __class__s&& r r(VivitTubeletEmbeddings.__init__0s ** && #-j(#C#CZ*Ia   ,q/ 1!} Q/ 1!} Q/ 1  %))   !3!3 r#cN<V^8dQhRS[PRS[P/#)r pixel_valuesreturntorchTensor)rrs"r r!r"@s'HHELLHU\\Hr#cVP^^4pVPV4P^4P^^4#)) transposer1flatten)r2r6s&&r forwardVivitTubeletEmbeddings.forward@s;#--a3 |,44Q7AA!QGGr#)r*r-r1) __name__ __module__ __qualname____firstlineno____doc__r(r?__static_attributes____classdictcell__ __classcell__r3rs@@r rr's&  HHHr#rcjaa]tRt^FtoRtV3RlV3RlltV3RlRltR V3RlRlltRtVt V;t #) VivitEmbeddingszQ Construct the CLS token, position and tubelet patch embeddings for video input. c <V^8dQhRS[/#rr)rrs"r r!VivitEmbeddings.__annotate__Ks ; ;{ ;r#c<\SV`4\P!\P !^^VP 44Vn\V4Vn VPPp\P!\P !^V^,VP 44Vn \P!VP4VnVPR,VnVPP"VnR#)r<r<NNN)r'r(r Parameterr9zerosr0 cls_tokenrpatch_embeddingsr-position_embeddingsDropouthidden_dropout_probdropoutr) patch_sizer*)r2rr-r3s&& r r(VivitEmbeddings.__init__Ks ekk!Q8J8J&KL 6v >++77 #%<< A{QPVPbPb0c#d zz&"<"<=  --b1//::r#cZ<V^8dQhRS[PRS[RS[RS[P/#)r embeddingsheightwidthr7)r9r:int)rrs"r r!rMWs;%D%D5<<%D%DUX%D]b]i]i%Dr#c VPPpVPP^,^, p\P P 4'gWE8XdW#8Xd VP#VPR,pVPR,pVPR,pW P^,,p W0P^,,p \VR,4p VP^WV4pVP^^^^4p\PPVW3RRR7pVP^^^^4P^RV4p\P!Wg3^R7#) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 g?bicubicF)sizemode align_cornersdim)NNN:Nr<N)rfrO)rSr-rTshaper9jit is_tracingrXrreshapepermuter functional interpolateviewcat) r2r[r\r]r- num_positionsclass_pos_embedpatch_pos_embedre new_height new_widthsqrt_num_positionss &&&& r interpolate_pos_encoding(VivitEmbeddings.interpolate_pos_encodingWsK++77 0066q9A= yy##%%+*F6?++ +22592259r"q11 __Q// &}c'9:)11!5G]`a)11!Q1=--33 ( 4 *11!Q1=BB1b#Nyy/;CCr#cT<V^8dQhRS[PRS[RS[P/#)rr6rwr7)r9r:bool)rrs"r r!rM~s-ELLD]b]i]ir#c  VPwr4rVpVPV4pVPPVRR4p \P !W3^R7pV'dWP WV4,pMzW`P^,8wgWpP^,8wd;\RV RV RVP^, RVP^, R2 4hWP,pVPV4pV#)r<rdzInput image size (*z) doesn't match model (z).rg) rhrSrRexpandr9rprwr* ValueErrorrTrW) r2r6rw batch_sizer,r/r\r]r[ cls_tokenss &&& r r?VivitEmbeddings.forward~s>J>P>P; e**<8 ^^**:r2> YY 7Q? ##&C&CJX]&^^J++u8J/J (%9+,Adooa.@-AE$&>&>>J\\*- r#)rRrWr*rSrXrT)F) rArBrCrDrEr(rwr?rFrGrHrIs@@r rKrKFs0 ; ;%D%DNr#rKcV^8dQhR\PR\PR\PR\PR\PR,R\R,R\R \ \ ,/#) rmodulequerykeyvalueattention_maskNscalingrWkwargs)rModuler9r:floatrr)rs"r r!r!s{%% II% <<% % << % LL4' % T\ %%'(%r#c VfVPR4R,p\P!WP^^44V,pVe W,p\P P VR\PR7PVP4p\P PWVPR7p\P!W4p V P^^4P4p W3#)N)redtype)ptrainingrg) rar9matmulr=rrmsoftmaxfloat32torrWr contiguous) rrrrrrrWr attn_weights attn_outputs &&&&&&&, r eager_attention_forwardrs**R.D(<<}}Q':;gEL!#4 ==((2U]](SVVW\WbWbcL==((6??([L,,|3K''1-88:K  $$r#cTaa]tRt^toV3RlV3RlltRV3RlRlltRtVtV;t#)VivitAttentionc <V^8dQhRS[/#rr)rrs"r r!VivitAttention.__annotate__s k k{ kr#c`<\SV`4WnVPVn\ VRVP VP,4VnVPVnVP R,Vn RVn \P!VP VPVP ,VPR7Vn\P!VP VPVP ,VPR7Vn\P!VP VPVP ,VPR7Vn\P!VPVP ,VP RR7VnR#)head_dimF)biasTNr)r'r(rnum_attention_headsgetattrr0rattention_probs_dropout_probattention_dropoutr is_causalrLinearqkv_biasq_projk_projv_projo_projr2rr3s&&r r(VivitAttention.__init__s*  #)#=#=  F4F4F&JdJd4de !'!D!D}}d* ii 2 2F4N4NQUQ^Q^4^eketetu ii 2 2F4N4NQUQ^Q^4^eketetu ii 2 2F4N4NQUQ^Q^4^eketetu ii : :T]] JFL^L^eij r#c <V^8dQhRS[PRS[PR,RS[S[,RS[S[PS[P3,/#r hidden_statesrNrr7)r9r:rrtuple)rrs"r r!rsX))||) t+)+, ) u||U\\) * )r#c VPRRp.VORNVPN5pVPV4PV4P ^^4pVP V4PV4P ^^4pVP V4PV4P ^^4p\P!VPP\4p T !VVVVV3RVP'gRM VPRVP/VBwrV P!.VORN5!P!4p VP#V 4p W3#)NrWrrg)rhrrror=rrr get_interfacer_attn_implementationrrrrrkrr) r2rrr input_shape hidden_shape query_states key_states value_statesattention_interfacerrs &&&, r r?VivitAttention.forwardsN $))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST (?(M(M KK , ,.E) %8      % $}}}C$2H2H % LL %  % ! "));;;;FFH kk+. ((r#) rrrrrrrrrrN rArBrCrDr(r?rFrGrHrIs@@r rrs  k k)))r#rcPaa]tRt^toV3RlV3RlltV3RlRltRtVtV;t#)VivitMLPc <V^8dQhRS[/#rr)rrs"r r!VivitMLP.__annotate__sKK{Kr#c(<\SV`4Wn\VP,Vn\ P!VPVP4Vn \ P!VPVP4Vn R#r) r'r(rr hidden_act activation_fnrrr0intermediate_sizefc1fc2rs&&r r(VivitMLP.__init__sb  #F$5$5699V//1I1IJ99V55v7I7IJr#cN<V^8dQhRS[PRS[P/#rrr7r8)rrs"r r!rs#U\\ellr#clVPV4pVPV4pVPV4pV#r)rrr)r2rs&&r r?VivitMLP.forwards4/ **=9 / r#)rrrrrrIs@@r rrs KKr#rcTaa]tRt^toV3RlV3RlltRV3RlRlltRtVtV;t#) VivitLayerc <V^8dQhRS[/#rr)rrs"r r!VivitLayer.__annotate__s>>{>r#cr<\SV`4\V4Vn\P !VP VPR7Vn\P !VP VPR7Vn \V4Vn \P!VP4VnR#)epsN)r'r(r attentionr LayerNormr0layer_norm_epslayernorm_beforelayernorm_afterrmlprUrVrWrs&&r r(VivitLayer.__init__sz '/ " V-?-?VEZEZ [!||F,>,>FDYDYZF#zz&"<"<= r#c<V^8dQhRS[PRS[PR,RS[S[,RS[P/#r)r9r:rr)rrs"r r!rsJ|| t++,   r#c TpVPV4pVP!W3/VBwrVPV4pW,pTpVPV4pVP V4pVPV4pW,pV#r)rrrWrr)r2rrrresidual_s&&&, r r?VivitLayer.forwards!--m< >>-R6R  ]3 %0 !,,]; /  ]3 %0 r#)rrWrrrrrrIs@@r rrs>>r#rcPaa]tRtRtoV3RlV3RlltV3RlRltRtVtV;t#) VivitPooleric <V^8dQhRS[/#rr)rrs"r r!VivitPooler.__annotate__s44{4r#c<\SV`4\P!VPVP 4Vn\VP,Vn R#r) r'r(rrr0pooler_output_sizedenser pooler_act activationrs&&r r(VivitPooler.__init__s> YYv1163L3LM  !2!23r#cN<V^8dQhRS[PRS[P/#rr8)rrs"r r!rs#U\\ellr#c\VR,pVPV4pVPV4pV#)rf)rfr)rr)r2rfirst_token_tensor pooled_outputs&& r r?VivitPooler.forwards2+40 #56  6 r#)rrrrIs@@r rrs44 r#rcaa]tRtRtoRtRtR tRtRR.tRt Rt Rt Rt Rt R]R]/tR t]P&!4V3R l4tV3R ltR tVtV;t#)VivitPreTrainedModeli!vivitr6TrKrr attentionsrSc <\SV`V4\V\4'dC\P !VP 4\P !VP4R#R#)zInitialize the weightsN)r' _init_weightsr+rKinitzeros_rRrT)r2rr3s&&r r"VivitPreTrainedModel._init_weights4sH f% fo . . KK(( ) KK22 3 /r#c&<V^8dQh/S[;R&#rr)rrs"r r!!VivitPreTrainedModel.__annotate__!s r#)video)rArBrCrDbase_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_can_compile_fullgraphrr_can_record_outputs_input_embed_layerr9no_gradr__annotate_func__rFrGrHrIs@@r rr!s $O!&*#*L9N"&!n, ]]_44)r#rcaa]tRtRtoR V3RlV3Rlllt]]!RR7]R V3RlRll444tRt Vt V;t #) VivitModeli=c&<V^8dQhRS[RS[/#)rradd_pooling_layer)rrz)rrs"r r!VivitModel.__annotate__?s  { t r#c <\SV`V4Wn\V4Vn\ P !\VP4Uu.uFp\V4NK up4Vn \ P!VPVPR7VnV'd \V4MRVnVP#4R#uupi)z^ add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer rN)r'r(rrKr[r ModuleListrangenum_hidden_layersrlayersrr0r layernormrpooler post_init)r2rr rr3s&&& r r(VivitModel.__init__?s   )&1mmvG_G_A`$aA`AZ%7A`$ab f&8&8f>S>ST->k&)D   %bsC F)tie_last_hidden_statesc <V^8dQhRS[PR,RS[RS[PR,RS[S[,RS[/#)rr6Nrwrrr7)r9 FloatTensorrzr:rrr )rrs"r r!rPse^j^j''$.^j#'^j t+ ^j +, ^j $ ^jr#c VPWR7p\VPVVR7pTpVPF pV!Wc3/VBpK VP V4pVP eVP V4MRp \ WR7#)a( Examples: ```python >>> import av >>> import numpy as np >>> from transformers import VivitImageProcessor, VivitModel >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len): ... ''' ... Sample a given number of frame indices from the video. ... Args: ... clip_len (`int`): Total number of frames to sample. ... frame_sample_rate (`int`): Sample every n-th frame. ... seg_len (`int`): Maximum allowed index of sample's last frame. ... Returns: ... indices (`list[int]`): List of sampled frame indices ... ''' ... converted_len = int(clip_len * frame_sample_rate) ... end_idx = np.random.randint(converted_len, seg_len) ... start_idx = end_idx - converted_len ... indices = np.linspace(start_idx, end_idx, num=clip_len) ... indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64) ... return indices >>> # video clip consists of 300 frames (10 seconds at 30 FPS) >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample 32 frames >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=1, seg_len=container.streams.video[0].frames) >>> video = read_video_pyav(container=container, indices=indices) >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400") >>> model = VivitModel.from_pretrained("google/vivit-b-16x2-kinetics400") >>> # prepare video for the model >>> inputs = image_processor(list(video), return_tensors="pt") >>> # forward pass >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 3137, 768] ```)rw)r inputs_embedsrN)last_hidden_state pooler_output)r[rrrrrr ) r2r6rwrrembedding_outputrlayersequence_outputrs &&&&, r r?VivitModel.forwardMsj ?? Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained position embeddings to the higher resolution. ) custom_introchaa]tRtRtoV3RlV3Rllt]]RV3RlRll44tRtVt V;t #)VivitForVideoClassificationic <V^8dQhRS[/#rr)rrs"r r!(VivitForVideoClassification.__annotate__s  { r#c8<\SV`V4VPVn\VRR7VnVP^8d,\ P !VPVP4M\ P!4Vn VP4R#)F)r N) r'r( num_labelsr rrrr0Identity classifierrrs&&r r($VivitForVideoClassification.__init__ss   ++%@ OUN_N_bcNc"))F$6$68I8IJikititiv r#c <V^8dQhRS[PR,RS[PR,RS[RS[S[,RS[/#)rr6Nlabelsrwrr7)r9r LongTensorrzrrr )rrs"r r!r's`i i ''$.i   4'i #' i +, i  i r#c VP!V3RV/VBpVPpVPVR,4pRpVeVP!W'VP3/VBp\ VVVP VPR7#)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> import av >>> import numpy as np >>> import torch >>> from transformers import VivitImageProcessor, VivitForVideoClassification >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len): ... ''' ... Sample a given number of frame indices from the video. ... Args: ... clip_len (`int`): Total number of frames to sample. ... frame_sample_rate (`int`): Sample every n-th frame. ... seg_len (`int`): Maximum allowed index of sample's last frame. ... Returns: ... indices (`list[int]`): List of sampled frame indices ... ''' ... converted_len = int(clip_len * frame_sample_rate) ... end_idx = np.random.randint(converted_len, seg_len) ... start_idx = end_idx - converted_len ... indices = np.linspace(start_idx, end_idx, num=clip_len) ... indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64) ... return indices >>> # video clip consists of 300 frames (10 seconds at 30 FPS) >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample 32 frames >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=4, seg_len=container.streams.video[0].frames) >>> video = read_video_pyav(container=container, indices=indices) >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400") >>> model = VivitForVideoClassification.from_pretrained("google/vivit-b-16x2-kinetics400") >>> inputs = image_processor(list(video), return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) ... logits = outputs.logits >>> # model predicts one of the 400 Kinetics-400 classes >>> predicted_label = logits.argmax(-1).item() >>> print(model.config.id2label[predicted_label]) LABEL_116 ```rwN)losslogitsrr)rfrrf)rrr+ loss_functionrr rr) r2r6r.rwroutputsr!r2r1s &&&&, r r?#VivitForVideoClassification.forwardsx$(:: $ 3K$ OU$ "33!9:  %%fdkkLVLD$!//))   r#)r+r)r)NNF) rArBrCrDr(rrr?rFrGrHrIs@@r r%r%s9  i i i i r#r%)Nr)r rr%).collections.abcrrr9rrr activationsr masking_utilsrmodeling_layersr modeling_outputsr r r modeling_utilsr rprocessing_utilsrutilsrrr utils.genericrrutils.output_capturingrconfiguration_vivitrrrrKrrrrrrr r%__all__rr#r rCs*/ &!69bbF&BBI5,HRYYH>LbiiL^%8.)RYY.)b ryy  +@ ")) 4?446pj%pjpjf  x "6x  x v Pr#