+
    œ°³jÚê  ã                   óh  € ^ RI Ht ^ RIHt ^ RIHt ^ RIt^ RIHt ^RIH	t
 ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHtHtHtHt ^RIHtHt ^RIHt ^RIH t  ^RI!H"t"H#t#H$t$H%t%H&t&H't' ^RI(H)t) ^RI*H+t+H,t, ^RI-H.t.H/t/H0t0 ^RI1H2t2H3t3H4t4 ]&Pj                  ! ]64      t7 ! R R]Pp                  4      t9 ! R R]Pp                  4      t:]$ ! R R]4      4       t;RBR R llt< ! R R]Pp                  4      t= ! R R]Pp                  4      t> ! R  R!]4      t? ! R" R#]Pp                  4      t@ ! R$ R%];4      tA ! R& R']Pp                  4      tB ! R( R)]Pp                  4      tC ! R* R+]Pp                  4      tD ! R, R-]Pp                  4      tE ! R. R/]Pp                  4      tF ! R0 R1]4      tG ! R2 R3]Pp                  4      tH ! R4 R5];4      tI]$! R6R77      ] ! R8 R9]"4      4       4       tJ]$! R:R77       ! R; R<];4      4       tK]$] ! R= R>]4      4       4       tL]$! R?R77       ! R@ RA];]4      4       tM. RCOtNR# )Dé    )ÚCallable)Ú	dataclass)ÚAnyN)Únn)Úinitialization)ÚACT2FN)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)Ú	AutoModelÚAutoModelForCausalLMÚAutoModelForSeq2SeqLM)ÚInstructBlipVideoConfigÚInstructBlipVideoQFormerConfigÚInstructBlipVideoVisionConfigc                   óf   a a€ ] tR t^;t oV3R lV 3R lltV3R lR ltRV3R lR lltRtVtV ;t	# )	Ú!InstructBlipVideoVisionEmbeddingsc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfig©r$   )ÚformatÚ__classdict__s   "€Ú�/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/instructblipvideo/modeling_instructblipvideo.pyÚ__annotate__Ú.InstructBlipVideoVisionEmbeddings.__annotate__<   s   ø€ ÷ cñ cÑ<ñ có    c                ó’  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! \        P                  ! ^^V P                  4      4      V n        \        P                  ! ^V P                  V P                  V P                  R7      V n        V P
                  V P                  ,          ^,          V n        V P                  ^,           V n        \        P                  ! \        P                  ! ^V P                  V P                  4      4      V n        R# )é   )Úin_channelsÚout_channelsÚkernel_sizeÚstrideN)ÚsuperÚ__init__r*   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	ParameterÚtorchÚrandnÚclass_embeddingÚConv2dÚpatch_embeddingÚnum_patchesÚnum_positionsÚposition_embedding©Úselfr*   Ú	__class__s   &&€r.   r9   Ú*InstructBlipVideoVisionEmbeddings.__init__<   sß   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|š|¬E¯KªK¸¸1¸d¿n¹nÓ,MÓNˆÔä!ŸyšyØ¨¯©ÀDÇOÁOÐ\`×\kÑ\kô 
ˆÔð !ŸO™O¨t¯©Õ>À1ÕDˆÔØ!×-Ñ-°Õ1ˆÔä"$§,¢,¬u¯{ª{¸1¸d×>PÑ>PÐRV×R`ÑR`Ó/aÓ"bˆÖr1   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r)   Ú
embeddingsÚheightÚwidthÚreturn©r?   ÚTensorÚint)r,   r-   s   "€r.   r/   r0   N   s;   ø€ ÷ &Dñ &D±5·<±<ð &DÉð &DÑUXð &DÑ]b×]iÑ]iñ &Dr1   c                óæ  € VP                   ^,          ^,
          pV P                  P                   ^,          ^,
          p\        P                  P	                  4       '       g   WE8X  d   W#8X  d   V P                  # V P                  R,          pV P                  R,          pVP                   R,          pW P
                  ,          p	W0P
                  ,          p
\        VR,          4      pVP                  ^W»V4      pVP                  ^ ^^^4      p\        P                  P                  VWš3RRR7      pVP                  ^ ^^^4      P                  ^RV4      p\        P                  ! Wg3^R7      # )	aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
g      à?ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)ºNNN:Nr3   N)rZ   :r3   NNéÿÿÿÿ)ÚshaperF   r?   ÚjitÚ
is_tracingr=   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)rH   rL   rM   rN   rD   rE   Úclass_pos_embedÚpatch_pos_embedrY   Ú
new_heightÚ	new_widthÚsqrt_num_positionss   &&&&        r.   Úinterpolate_pos_encodingÚ:InstructBlipVideoVisionEmbeddings.interpolate_pos_encodingN   sE  € ð !×&Ñ& qÕ)¨AÕ-ˆØ×/Ñ/×5Ñ5°aÕ8¸1Õ<ˆô �y‰y×#Ñ#×%Ò%¨+Ô*FÈ6Ì?Ø×*Ñ*Ð*à×1Ñ1°%Õ8ˆØ×1Ñ1°%Õ8ˆà×Ñ˜rÕ"ˆàŸ™Õ.ˆ
ØŸ_™_Õ,ˆ	ä& }°cÕ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÔCÐCr1   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r)   Úpixel_valuesrj   rO   )r?   ÚFloatTensorÚboolrQ   )r,   r-   s   "€r.   r/   r0   v   s0   ø€ ÷ ñ ¡E×$5Ñ$5ð ÑQUð Ñbg×bnÑbnñ r1   c                ó$  € VP                   w  r4rVV P                  P                  P                  pV P                  VP	                  VR 7      4      pVP                  ^4      P                  ^^4      pV P                  P                  V^R4      P	                  V4      p	\        P                  ! W˜.^R7      p
V'       d   V P                  W¥V4      pMV P                  pW«RRV
P                  ^4      1R3,          P	                  V4      ,           p
V
# ))ÚdtyperX   rZ   Nr[   )r\   rC   Úweightrq   ÚtoÚflattenÚ	transposerA   Úexpandr?   rd   rj   rF   rU   )rH   rm   rj   Ú
batch_sizeÚ_rM   rN   Útarget_dtypeÚpatch_embedsÚclass_embedsrL   rF   s   &&&         r.   ÚforwardÚ)InstructBlipVideoVisionEmbeddings.forwardv   sñ   € Ø'3×'9Ñ'9Ñ$ˆ
�vØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆØ×+Ñ+×2Ñ2°:¸qÀ"ÓE×HÑHÈÓVˆÜ—Y’Y Ð;ÀÔCˆ
ß#Ø!%×!>Ñ!>¸zÐSXÓ!YÑà!%×!8Ñ!8ÐØ°QÐ8L¸*¿/¹/È!Ó:LÐ8LÈaÐ5OÕ"P×"SÑ"SÐT`Ó"aÕaˆ
ØÐr1   )	rA   r*   r;   r<   rD   rE   rC   r=   rF   ©F)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r9   rj   r|   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rI   r-   s   @@r.   r&   r&   ;   s-   ù‡ € ÷có c÷$&Dð &D÷P÷ ò r1   r&   c                   ó@   a a€ ] tR t^…t oRtV 3R ltRR ltRtVtV ;t	# )Ú"InstructBlipVideoQFormerEmbeddingsz;Construct the embeddings from word and position embeddings.c                óH  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P                   ! \"        P$                  ! VP                  4      P'                  R4      RR7      V n        Wn        R# ))Úpadding_idx©ÚepsF)Ú
persistentN©r3   r[   )r8   r9   r   Ú	EmbeddingÚ
vocab_sizer:   Úpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚ	layernormÚDropoutÚhidden_dropout_probÚdropoutÚBufferr?   Úarangerv   Úposition_idsr*   rG   s   &&€r.   r9   Ú+InstructBlipVideoQFormerEmbeddings.__init__ˆ   sÀ   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔàŽr1   c                ó  € Ve   VP                  4       ^,          pM^ pVf,   V P                  RWEV,           13,          P                  4       pVeb   V P                  V4      pV P	                  VP                  VP                  4      4      pWg,           pVe   \        P                  ! W63^R7      pMTpVP                  V P                  P                  P                  4      pV P                  V4      pV P                  V4      pV# )NrZ   rX   )rU   r�   Úcloner’   r”   rs   Údevicer?   rd   r—   rr   rq   rš   )rH   Ú	input_idsr�   Úquery_embedsÚpast_key_values_lengthÚ
seq_lengthrL   r”   s   &&&&&   r.   r|   Ú*InstructBlipVideoQFormerEmbeddings.forward•   së   € ð Ò Ø"Ÿ™Ó)¨!Õ,‰JàˆJàÒØ×,Ñ,¨QÐ0FÐVlÕIlÐ0lÐ-lÕm×sÑsÓuˆLàÒ Ø×-Ñ-¨iÓ8ˆJà"&×":Ñ":¸<¿?¹?È:×K\ÑK\Ó;]Ó"^ÐØ#Õ9ˆJàÒ'Ü"ŸYšY¨Ð'AÀqÔI�
øà%ˆJà—]‘] 4§>¡>×#8Ñ#8×#>Ñ#>Ó?ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr1   )r*   rš   r—   r”   r�   r’   ©NNNr   )
r   r€   r�   r‚   Ú__doc__r9   r|   rƒ   r„   r…   r†   s   @@r.   rˆ   rˆ   …   s   ù‡ € ÙEõ÷ô r1   rˆ   c                   óŽ   a a€ ] tR t^µt oRtRtRtRtRtRt	Rt
Rt. ROt]P                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )Ú InstructBlipVideoPreTrainedModelÚblipTc                óx  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  RVR7       \        P                  ! VP                  RVR7       R# \	        V\        \        34      '       d#   \        P                  ! VP                  4       R# \	        V\        4      '       db   \        P                  ! VP                   \"        P$                  ! VP                   P&                  R,          4      P)                  R4      4       R# R# )zInitialize the weightsç        )ÚmeanÚstdNr[   rŽ   )r8   Ú_init_weightsr*   Úinitializer_rangeÚ
isinstancer&   ÚinitÚtrunc_normal_rF   rA   Ú)InstructBlipVideoForConditionalGenerationÚInstructBlipVideoModelÚzeros_Úquery_tokensrˆ   Úcopy_r�   r?   rœ   r\   rv   )rH   ÚmoduleÚfactorrI   s   && €r.   r°   Ú.InstructBlipVideoPreTrainedModel._init_weightsË   sÔ   ø€ ô 	‰Ñ˜fÔ%Ø—‘×.Ñ.ˆÜ�fÔ?×@Ò@Ü×Ò˜v×8Ñ8¸sÈÕOÜ×Ò˜v×5Ñ5¸CÀV×LÜ˜Ô!JÔLbÐ c×dÒdÜ�KŠK˜×+Ñ+Ö,Ü˜Ô B×CÒCÜ�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhñ Dr1   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r(   ©r"   )r,   r-   s   "€r.   r/   Ú-InstructBlipVideoPreTrainedModel.__annotate__µ   s   ø‡ ‚ á#Ñ#ò r1   © )ÚvideoÚtext)rˆ   ÚInstructBlipVideoAttentionÚInstructBlipVideoEncoderLayerÚInstructBlipVideoQFormerLayerÚ*InstructBlipVideoQFormerMultiHeadAttentionÚ"InstructBlipVideoQFormerSelfOutput)r   r€   r�   r‚   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_no_split_modulesr?   Úno_gradr°   Ú__annotate_func__rƒ   r„   r…   r†   s   @@r.   rª   rª   µ   sa   ù‡ € ð ÐØ(ÐØ&*Ð#Ø"&ÐØÐØ€NØÐà!ÐòÐð ‡]‚]ƒ_ô
ió ð
i÷/ … r1   rª   c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r)   rº   ÚqueryÚkeyÚvalueÚattention_maskNÚscalingrš   )r   ÚModuler?   rQ   Úfloat)r,   s   "r.   r/   r/   Ú   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r1   c                 ó~  € \         P                  ! WP                  RR4      4      V,          pVe	   W„,           p\        P                  P                  VRR7      p\        P                  P                  W†V P                  R7      p\         P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )r3   rX   )ÚpÚtrainingr[   éþÿÿÿ)	r?   Úmatmulru   r   ra   Úsoftmaxrš   rÝ   Ú
contiguous)
rº   rÔ   rÕ   rÖ   r×   rØ   rš   ÚkwargsÚattn_weightsÚattn_outputs
   &&&&&&&,  r.   Úeager_attention_forwardrå   Ú   s˜   € ô —<’< §}¡}°R¸Ó'<Ó=ÀÕG€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r1   c                   óZ   a a€ ] tR t^ñt oRtV 3R ltV3R lR ltV3R lR ltRtVt	V ;t
# )rÃ   z=Multi-headed attention from 'Attention Is All You Need' paperc                ó<  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  V P                  ,          V P                  8w  d'   \        R V P                   RV P                   R24      hV P                  R,          V n	        RV n
        VP                  V n        \        P                  ! V P                  ^V P                  ,          RR7      V n        VP                  '       dj   \        P                   ! \"        P$                  ! V P                  4      4      p\        P                   ! \"        P$                  ! V P                  4      4      pMRpRpVeU   \"        P&                  ! V\"        P(                  ! VRR7      V34      p\        P                   ! V4      V P                  n        \        P                  ! V P                  V P                  4      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).F)ÚbiasN)Úrequires_gradç      à¿)r8   r9   r*   r:   r;   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚ	is_causalÚattention_dropoutr   ÚLinearÚqkvÚqkv_biasr>   r?   Úzerosrd   Ú
zeros_likerè   Ú
projection)rH   r*   Úq_biasÚv_biasrô   rI   s   &&   €r.   r9   Ú#InstructBlipVideoAttention.__init__ô   su  ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Õ8ˆŒØ�=‰=˜4Ÿ>™>Õ)¨T¯^©^Ô;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÕ(ˆŒ
ØˆŒØ!'×!9Ñ!9ˆÔô —9’9˜TŸ^™^¨Q°·±Õ-?ÀeÔLˆŒà�?�?ˆ?Ü—\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆFÜ—\’\¤%§+¢+¨d¯n©nÓ"=Ó>‰FàˆFØˆFàÒÜ—y’y &¬%×*:Ò*:¸6ÐQVÔ*WÐY_Ð!`ÓaˆHÜŸLšL¨Ó2ˆD�H‰HŒMäŸ)š) D§N¡N°D·N±NÓCˆŽr1   c                ó@   <€ V ^8„  d   QhRS[ P                  RS[RS[/# )r)   ÚtensorÚseq_lenÚbszrP   )r,   r-   s   "€r.   r/   Ú'InstructBlipVideoAttention.__annotate__  s+   ø€ ÷ eñ e™UŸ\™\ð e±Cð e¹cñ er1   c                óŒ   € VP                  W2V P                  V P                  4      P                  ^^4      P	                  4       # )r3   )rc   rì   rí   ru   rá   )rH   rü   rý   rþ   s   &&&&r.   Ú_shapeÚ!InstructBlipVideoAttention._shape  s5   € Ø�{‰{˜3¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr1   c                ó¼   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r)   Úhidden_statesrO   N)r?   rQ   Útuple)r,   r-   s   "€r.   r/   rÿ     sJ   ø€ ÷ ")ñ ")á—|‘|ð")ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ	")r1   c           	     ó>  € VP                  4       w  r4pV P                  V4      pVP                  W4^V P                  WPP                  ,          4      P	                  ^^ ^^^4      pV^ ,          V^,          V^,          r˜p\
        P                  ! V P                  P                  \        4      p
T
! V VVV	3RRRV P                  '       g   RMV P                  RV P                  /VB w  r¼VP                  W4R4      P                  4       pV P                  V4      pW¼3# )z#Input shape: Batch x Time x Channelr×   Nrš   r­   rØ   r[   )rU   ró   r_   rì   r`   r   Úget_interfacer*   Ú_attn_implementationrå   rÝ   rñ   rï   rá   r÷   )rH   r  râ   rþ   Útgt_lenr;   Ú	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerä   rã   s   &&,          r.   r|   Ú"InstructBlipVideoAttention.forward  s$  € ð #0×"4Ñ"4Ó"6Ñˆ�ià—H‘H˜]Ó+ˆ	à×%Ñ% c°A°t·~±~Ày×TbÑTbÕGbÓc×kÑkØˆq�!�Q˜ó
ˆ	ð 2;¸1µ¸yÈ½|ÈYÐWXÍ\ ,ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØñ		%
ð
  ð	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —J‘Jð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#¸Ó;×FÑFÓHˆØ—o‘o kÓ2ˆàÐ(Ð(r1   )	rñ   r*   r;   rí   rð   rì   r÷   ró   rï   )r   r€   r�   r‚   r¨   r9   r  r|   rƒ   r„   r…   r†   s   @@r.   rÃ   rÃ   ñ   s)   ù‡ € ÙGõD÷>eð e÷")÷ ")ð ")r1   rÃ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInstructBlipVideoMLPi;  c                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # ©N)r8   r9   r*   r   Ú
hidden_actÚactivation_fnr   rò   r:   Úintermediate_sizeÚfc1Úfc2rG   s   &&€r.   r9   ÚInstructBlipVideoMLP.__init__<  sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r)   r  rO   ©r?   rQ   )r,   r-   s   "€r.   r/   Ú!InstructBlipVideoMLP.__annotate__C  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r1   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r  )r  r  r  ©rH   r  s   &&r.   r|   ÚInstructBlipVideoMLP.forwardC  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr1   )r  r*   r  r  ©	r   r€   r�   r‚   r9   r|   rƒ   r„   r…   r†   s   @@r.   r  r  ;  s   ù‡ € õK÷÷ ð r1   r  c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )rÄ   iJ  c                ó    <€ V ^8„  d   QhRS[ /# r(   r¾   )r,   r-   s   "€r.   r/   Ú*InstructBlipVideoEncoderLayer.__annotate__K  s   ø€ ÷ Sñ SÑ6ñ Sr1   c                óJ  <€ \         SV `  4        VP                  V n        \	        V4      V n        \        P                  ! V P                  VP                  R 7      V n	        \        V4      V n        \        P                  ! V P                  VP                  R 7      V n        R# ©r‹   N)r8   r9   r:   r;   rÃ   Ú	self_attnr   r•   r–   Úlayer_norm1r  ÚmlpÚlayer_norm2rG   s   &&€r.   r9   Ú&InstructBlipVideoEncoderLayer.__init__K  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ3°FÓ;ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÔÜ'¨Ó/ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÖr1   c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r)   r  râ   rO   )r?   rQ   r   r   rn   )r,   r-   s   "€r.   r/   r$  T  s9   ø€ ÷ ñ á—|‘|ðñ Ñ+Õ,ðñ 
×	Ñ	ñ	r1   c                óÀ   € TpV P                  V4      pV P                  ! RR V/VB w  rW,           pTpV P                  V4      pV P                  V4      pW,           pV# )r  rÀ   )r(  r'  r*  r)  )rH   r  râ   Úresidualrx   s   &&,  r.   r|   Ú%InstructBlipVideoEncoderLayer.forwardS  su   € ð !ˆà×(Ñ(¨Ó7ˆØŸ>š>ñ 
Ø'ð
àñ
Ñˆð &Õ0ˆØ ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆà%Õ0ˆàÐr1   )r;   r(  r*  r)  r'  )
r   r€   r�   r‚   r9   r   r|   rƒ   r„   r…   r†   s   @@r.   rÄ   rÄ   J  s*   ù‡ € ÷Só Sð ÷ó ÷ð r1   rÄ   c                   ó^   a a€ ] tR tRt oRtV3R lV 3R llt]V3R lR l4       tRtVt	V ;t
# )ÚInstructBlipVideoEncoderij  a
  
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`InstructBlipVideoEncoderLayer`].

Args:
    config (`InstructBlipVideoConfig`):
        The corresponding vision configuration for the `InstructBlipVideoEncoder`.
c                ó    <€ V ^8„  d   QhRS[ /# r(   r¾   )r,   r-   s   "€r.   r/   Ú%InstructBlipVideoEncoder.__annotate__t  s   ø€ ÷ ,ñ ,Ñ6ñ ,r1   c                óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        R V n	        R# u upi ©FN)
r8   r9   r*   r   Ú
ModuleListÚrangeÚnum_hidden_layersrÄ   ÚlayersÚgradient_checkpointing)rH   r*   rx   rI   s   && €r.   r9   Ú!InstructBlipVideoEncoder.__init__t  sU   ø€ Ü‰ÑÔØŒÜ—m’mÔTYÐZ`×ZrÑZrÔTsÓ$tÑTsÈqÔ%BÀ6Ö%JÑTsÑ$tÓuˆŒØ&+ˆÖ#ùò %uó   ½A&c                óF   <€ V ^8„  d   QhRS[ S[,          RS[S[,          /# )r)   râ   rO   )r   r   r  r   )r,   r-   s   "€r.   r/   r3  {  s0   ø€ ÷ @ñ @ñ Ñ+Õ,ð@ñ 
‘Õ	 ñ	@r1   c                óV   € TpV P                    F  pV! V3/ VB pK  	  \        VR 7      # )©Úlast_hidden_state)r9  r   )rH   Úinputs_embedsrâ   r  Úencoder_layers   &&,  r.   r|   Ú InstructBlipVideoEncoder.forwardz  s9   € ð &ˆØ!Ÿ[œ[ˆMÙ)ØñàñŠMñ )ô °Ô?Ð?r1   )r*   r:  r9  )r   r€   r�   r‚   r¨   r9   r   r|   rƒ   r„   r…   r†   s   @@r.   r1  r1  j  s0   ù‡ € ñ÷,ó ,ð ÷@ó ÷@ð @r1   r1  c                   ó¦   a a€ ] tR tRt oRtRtR]R]/tV3R lV 3R llt	]
]! RR	7      ]RV3R
 lR ll4       4       4       tR tV3R ltRtVtV ;t# )ÚInstructBlipVideoVisionModeliŠ  rm   rÁ   r  Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# r(   r+   )r,   r-   s   "€r.   r/   Ú)InstructBlipVideoVisionModel.__annotate__“  s   ø€ ÷ 	ñ 	Ñ<ñ 	r1   c                óö   <€ \         SV `  V4       Wn        VP                  p\	        V4      V n        \        V4      V n        \        P                  ! W!P                  R 7      V n        V P                  4        R# r&  )r8   r9   r*   r:   r&   rL   r1  Úencoderr   r•   r–   Úpost_layernormÚ	post_init)rH   r*   r;   rI   s   && €r.   r9   Ú%InstructBlipVideoVisionModel.__init__“  sY   ø€ Ü‰Ñ˜Ô ØŒØ×&Ñ&ˆ	ä;¸FÓCˆŒÜ/°Ó7ˆŒÜ Ÿlšl¨9×:OÑ:OÔPˆÔà�‰Ör1   F)Útie_last_hidden_statesc          	      ót   <€ V ^8„  d   QhRS[ P                  R,          RS[RS[S[,          RS[S[,          /# )r)   rm   Nrj   râ   rO   )r?   rn   ro   r   r   r  r   )r,   r-   s   "€r.   r/   rH  ¡  sJ   ø€ ÷ 
ñ 
á×'Ñ'¨$Õ.ð
ñ #'ð
ñ Ñ+Õ,ð	
ñ
 
Ñ+Õ	+ñ
r1   c                óô   € Vf   \        R4      hV P                  WR7      pV P                  ! RRV/VB pVP                  pV P	                  V4      pVR,          pV P	                  V4      p\        VVR7      # )Nz You have to specify pixel_values)rj   rA  ©r@  Úpooler_outputrÀ   ©rZ   r   rZ   )rî   rL   rJ  r@  rK  r   )rH   rm   rj   râ   r  Úencoder_outputsr@  Úpooled_outputs   &&&,    r.   r|   Ú$InstructBlipVideoVisionModel.forwardž  s‘   € ð ÒÜÐ?Ó@Ð@àŸ™¨˜Óhˆà+/¯<ª<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ ×/Ñ/Ð0AÓBÐà)¨'Õ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
r1   c                ó   € V P                   # r  )rL   ©rH   s   &r.   Úget_input_embeddingsÚ1InstructBlipVideoVisionModel.get_input_embeddings¼  s   € Ø�‰Ðr1   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r(   r+   )r,   r-   s   "€r.   r/   rH  Š  s   ø‡ ‚ ñ *Ñ)ò r1   )r*   rL   rJ  rK  ©NF)r   r€   r�   r‚   Úmain_input_namerÉ   rÄ   rÃ   Ú_can_record_outputsr9   r   r   r   r|   rY  rÒ   rƒ   r„   r…   r†   s   @@r.   rE  rE  Š  sk   ù‡ € Ø$€OØÐð 	Ð6ØÐ0ðÐ÷
	ó 	ð  Ù¨EÔ2Ø÷
ñ 
ó ó 3ó  ð
ò6÷e … r1   rE  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )rÆ   iÀ  c                óø  <€ \         SV `  4        Wn        VP                  VP                  ,          ^ 8w  d;   \        VR4      '       g)   \        RVP                  VP                  3,          4      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        RV n        VP                  V n        \        P                  ! VP                  V P                  4      V n        V'       dc   \        P                  ! VP"                  V P                  4      V n        \        P                  ! VP"                  V P                  4      V n        R# \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        R# )r   Úembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)FNrê   )r8   r9   r*   r:   rë   Úhasattrrî   rR   Úattention_head_sizeÚall_head_sizerØ   rð   Úattention_probs_dropout_probrñ   r   rò   rÔ   Úencoder_hidden_sizerÕ   rÖ   ©rH   r*   Úis_cross_attentionrI   s   &&&€r.   r9   Ú3InstructBlipVideoQFormerMultiHeadAttention.__init__Á  sk  ø€ Ü‰ÑÔØŒØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ^Ø×%Ñ% v×'AÑ'AÐBõCóð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒØˆŒØ!'×!DÑ!DˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ßÜ—y’y ×!;Ñ!;¸T×=OÑ=OÓPˆDŒHÜŸš 6×#=Ñ#=¸t×?QÑ?QÓRˆDŽJä—y’y ×!3Ñ!3°T×5GÑ5GÓHˆDŒHÜŸš 6×#5Ñ#5°t×7IÑ7IÓJˆDŽJr1   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r)   râ   ©r   r   )r,   r-   s   "€r.   r/   Ú7InstructBlipVideoQFormerMultiHeadAttention.__annotate__Ù  s   ø€ ÷ ,)ñ ,)ñ Ñ+Õ,ñ,)r1   c                ó  € VR JpVP                   R R p. VORNV P                  N5pV'       d   Tp	TpMTp	. V	P                   R R ORNV P                  N5p
V P                  V4      P                  V4      P	                  ^^4      pV P                  V	4      P                  V
4      P	                  ^^4      pV P                  V	4      P                  V
4      P	                  ^^4      p\        P                  ! V P                  P                  \        4      pT! V VVVV3RV P                  '       g   RMV P                  RV P                  /VB w  ppVP                  ! . VP                   R,          ORN5!  P!                  4       pVV3# )Nrš   r­   rØ   :Nr)   Nr[   )r\   rc  rÔ   rc   ru   rÕ   rÖ   r   r  r*   r  rå   rÝ   rñ   rØ   r_   rá   )rH   r  r×   Úencoder_hidden_statesÚencoder_attention_maskrâ   rh  Úinput_shapeÚhidden_shapeÚcurrent_statesÚkv_shapeÚquery_layerÚ	key_layerÚvalue_layerr  rä   rã   s   &&&&&,           r.   r|   Ú2InstructBlipVideoQFormerMultiHeadAttention.forwardÙ  s“  € ð 3¸$Ð>Ðà#×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆçØ2ˆNØ3‰Nà*ˆNàM�^×)Ñ)¨#¨2Ð.ÐM°ÐM°D×4LÑ4LÑMˆØ—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆØ—H‘H˜^Ó,×1Ñ1°(Ó;×EÑEÀaÈÓKˆ	Ø—j‘j Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)ÐE¨;×+<Ñ+<¸RÕ+@ÐEÀ"ÓE×PÑPÓRˆØ˜LÐ(Ð(r1   )
rd  rñ   rc  r*   rð   rÕ   rë   rÔ   rØ   rÖ   r~   ©NNNr!  r†   s   @@r.   rÆ   rÆ   À  s   ù‡ € ÷K÷0,)÷ ,)ò ,)r1   rÆ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )rÇ   i  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# r&  )r8   r9   r   rò   r:   Údenser•   r–   r˜   r™   rš   rG   s   &&€r.   r9   Ú+InstructBlipVideoQFormerSelfOutput.__init__	  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr1   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r)   r  Úinput_tensorrO   r  )r,   r-   s   "€r.   r/   Ú/InstructBlipVideoQFormerSelfOutput.__annotate__  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ r1   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# r  ©r|  rš   r•   ©rH   r  r€  s   &&&r.   r|   Ú*InstructBlipVideoQFormerSelfOutput.forward  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr1   ©r•   r|  rš   r!  r†   s   @@r.   rÇ   rÇ     ó   ù‡ € õ>÷÷ ð r1   rÇ   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )Ú!InstructBlipVideoQFormerAttentioni  c                ód   <€ \         SV `  4        \        W4      V n        \	        V4      V n        R # r  )r8   r9   rÆ   Ú	attentionrÇ   Úoutputrg  s   &&&€r.   r9   Ú*InstructBlipVideoQFormerAttention.__init__  s&   ø€ Ü‰ÑÔÜCÀFÓ_ˆŒÜ8¸Ó@ˆŽr1   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )r)   r  r×   Nro  rp  râ   rO   )r?   rQ   rn   r   r   )r,   r-   s   "€r.   r/   Ú.InstructBlipVideoQFormerAttention.__annotate__  sv   ø€ ÷  ñ  á—|‘|ð ñ ×)Ñ)¨DÕ0ð ñ  %×0Ñ0°4Õ7ð	 ñ
 !&× 1Ñ 1°DÕ 8ð ñ Ñ+Õ,ð ñ 
�‰ñ r1   c                ó`   € V P                   ! RR VRVRVRV/VB w  rgV P                  Wa4      pV# )r  r×   ro  rp  rÀ   ©r�  rŽ  )	rH   r  r×   ro  rp  râ   rä   rx   Úattention_outputs	   &&&&&,   r.   r|   Ú)InstructBlipVideoQFormerAttention.forward  sY   € ð Ÿšñ 
Ø'ð
à)ð
ð #8ð
ð $:ð	
ð
 ñ
‰ˆð  Ÿ;™; {ÓBÐØÐr1   r“  r~   ry  r!  r†   s   @@r.   r‹  r‹    s   ù‡ € ÷A÷
 ÷  ò  r1   r‹  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú$InstructBlipVideoQFormerIntermediatei/  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # r  )r8   r9   r   rò   r:   r  r|  r²   r  Ústrr   Úintermediate_act_fnrG   s   &&€r.   r9   Ú-InstructBlipVideoQFormerIntermediate.__init__0  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  r  )r,   r-   s   "€r.   r/   Ú1InstructBlipVideoQFormerIntermediate.__annotate__8  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r1   c                óJ   € V P                  V4      pV P                  V4      pV# r  ©r|  rš  r  s   &&r.   r|   Ú,InstructBlipVideoQFormerIntermediate.forward8  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr1   rŸ  r!  r†   s   @@r.   r—  r—  /  s   ù‡ € õ9÷÷ ð r1   r—  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInstructBlipVideoQFormerOutputi>  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# r&  )r8   r9   r   rò   r  r:   r|  r•   r–   r˜   r™   rš   rG   s   &&€r.   r9   Ú'InstructBlipVideoQFormerOutput.__init__?  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr1   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# r  r  )r,   r-   s   "€r.   r/   Ú+InstructBlipVideoQFormerOutput.__annotate__E  r‚  r1   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# r  r„  r…  s   &&&r.   r|   Ú&InstructBlipVideoQFormerOutput.forwardE  r‡  r1   rˆ  r!  r†   s   @@r.   r¢  r¢  >  r‰  r1   r¢  c                   óT   a a€ ] tR tRt oV 3R ltRV3R lR lltR tR tRtVt	V ;t
# )	rÅ   iL  c                óp  <€ \         SV `  4        VP                  V n        ^V n        \	        V4      V n        W n        W!P                  ,          ^ 8X  d   \	        VRR7      V n        RV n	        MRV n	        \        V4      V n        \        V4      V n        \        V4      V n        \        V4      V n        R# )r3   T)rh  FN)r8   r9   Úchunk_size_feed_forwardÚseq_len_dimr‹  r�  Ú	layer_idxÚcross_attention_frequencyÚcrossattentionÚhas_cross_attentionr—  Úintermediater¢  rŽ  Úintermediate_queryÚoutput_query©rH   r*   r­  rI   s   &&&€r.   r9   Ú&InstructBlipVideoQFormerLayer.__init__M  s™   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ:¸6ÓBˆŒà"Œà×7Ñ7Õ7¸1Ô<Ü"CÀFÐ_cÔ"dˆDÔØ'+ˆDÕ$à',ˆDÔ$ä@ÀÓHˆÔÜ4°VÓ<ˆŒä"FÀvÓ"NˆÔÜ:¸6ÓBˆÖr1   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rk  rl  )r,   r-   s   "€r.   r/   Ú*InstructBlipVideoQFormerLayer.__annotate__a  s   ø€ ÷ 3ñ 3ñ Ñ+Õ,ñ3r1   c           	     ó†  € V P                   ! V3R V/VB pV^ 8”  dù   VRRV1R3,          pV P                  '       d)   Vf   \        R4      hV P                  ! V3R VRVRV/VB p\	        V P
                  V P                  V P                  V4      p	VP                  ^,          V8”  dk   \	        V P                  V P                  V P                  VRVR1R3,          4      P                  V	P                  4      p
\        P                  ! Wš.^R7      p	V	# \	        V P                  V P                  V P                  V4      p	V	# )r×   rZ   Nz>encoder_hidden_states must be given for cross-attention layersro  rp  rX   )r�  r°  rî   r¯  r   Úfeed_forward_chunk_queryr«  r¬  r\   Úfeed_forward_chunkrs   r¡   r?   rd   )rH   r  r×   ro  rp  Úquery_lengthrâ   r”  Úquery_attention_outputÚlayer_outputÚlayer_output_texts   &&&&&&,    r.   r|   Ú%InstructBlipVideoQFormerLayer.forwarda  s|  € ð  Ÿ>š>Øñ
à)ð
ð ñ
Ðð ˜!ÔØ%5°a¸¸,¸ÈÐ6IÕ%JÐ"à×'×'Ð'Ø(Ò0Ü$Ð%eÓfÐfØ)-×)<Ò)<Ø*ñ*à#1ð*ð +@ð*ð ,Bð	*ð
 ñ*Ð&ô 5Ø×-Ñ-Ø×,Ñ,Ø× Ñ Ø&ó	ˆLð  ×%Ñ% aÕ(¨<Ô7Ü$=Ø×+Ñ+Ø×0Ñ0Ø×$Ñ$Ø$ Q¨©°qÐ%8Õ9ó	%÷
 ‘"�\×(Ñ(Ó)ð "ô  %Ÿyšy¨,Ð)JÐPQÔR�ð Ðô 5Ø×'Ñ'Ø×,Ñ,Ø× Ñ Ø ó	ˆLð Ðr1   c                óJ   € V P                  V4      pV P                  W!4      pV# r  )r±  rŽ  ©rH   r”  Úintermediate_outputr½  s   &&  r.   rº  Ú0InstructBlipVideoQFormerLayer.feed_forward_chunk–  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐr1   c                óJ   € V P                  V4      pV P                  W!4      pV# r  )r²  r³  rÁ  s   &&  r.   r¹  Ú6InstructBlipVideoQFormerLayer.feed_forward_chunk_query›  s+   € Ø"×5Ñ5Ð6FÓGÐØ×(Ñ(Ð)<ÓOˆØÐr1   )
r�  r«  r¯  r°  r±  r²  r­  rŽ  r³  r¬  r§   )r   r€   r�   r‚   r9   r|   rº  r¹  rƒ   r„   r…   r†   s   @@r.   rÅ   rÅ   L  s%   ù‡ € õC÷(3ò 3òj÷
ò r1   rÅ   c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚInstructBlipVideoQFormerEncoderi¡  c           	     óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        R V n	        R# u upi r5  )
r8   r9   r*   r   r6  r7  r8  rÅ   Úlayerr:  r´  s   && €r.   r9   Ú(InstructBlipVideoQFormerEncoder.__init__¢  sY   ø€ Ü‰ÑÔØŒÜ—]’]ÜOTÐU[×UmÑUmÔOnÓoÑOnÀ)Ô*¨6Ö=ÑOnÑoó
ˆŒ
ð ',ˆÖ#ùò pr<  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rk  rl  )r,   r-   s   "€r.   r/   Ú,InstructBlipVideoQFormerEncoder.__annotate__«  s   ø€ ÷ 
ñ 
ñ Ñ+Õ,ñ
r1   c                óª   € \        V P                  P                  4       F%  pV P                  V,          pV! VVV3R VRV/VB pK'  	  \	        VR7      # )rp  r»  r?  )r7  r*   r8  rÉ  r   )	rH   r  r×   ro  rp  r»  râ   ÚiÚlayer_modules	   &&&&&&,  r.   r|   Ú'InstructBlipVideoQFormerEncoder.forwardª  sm   € ô �t—{‘{×4Ñ4Ö5ˆAØŸ:™: a�=ˆLá(ØØØ%ñð (>ð	ð
 *ðð ñŠMñ 6ô 9Ø+ô
ð 	
r1   )r*   r:  rÉ  r§   )
r   r€   r�   r‚   r9   r   r|   rƒ   r„   r…   r†   s   @@r.   rÇ  rÇ  ¡  s(   ù‡ € õ,ð ÷
ñ 
ó ÷
ð 
r1   rÇ  c                   óÊ   a a€ ] tR tRt oRtRtRtRtRtR]	R]
! ]^RR7      .R]
! ]^R	R7      ./tV3R
 lV 3R lltR tR t]]]RV3R lR ll4       4       4       tRtVtV ;t# )ÚInstructBlipVideoQFormerModeliÅ  z†
Querying Transformer (Q-Former), used in InstructBlipVideo. Slightly modified from BLIP-2 as it also takes the
instruction as input.
Tr  rF  z
.attention)ÚindexÚ
layer_nameÚcross_attentionsz.crossattentionc                ó    <€ V ^8„  d   QhRS[ /# r(   )r#   )r,   r-   s   "€r.   r/   Ú*InstructBlipVideoQFormerModel.__annotate__Ú  s   ø€ ÷ ñ Ñ=ñ r1   c                ó’   <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        V P                  4        R # r  )r8   r9   r*   rˆ   rL   rÇ  rJ  rL  rG   s   &&€r.   r9   Ú&InstructBlipVideoQFormerModel.__init__Ú  s7   ø€ Ü‰Ñ˜Ô ØŒä<¸VÓDˆŒä6°vÓ>ˆŒà�‰Ör1   c                ó.   € V P                   P                  # r  ©rL   r’   rX  s   &r.   rY  Ú2InstructBlipVideoQFormerModel.get_input_embeddingsä  s   € Ø�‰×.Ñ.Ð.r1   c                ó&   € WP                   n        R # r  rÛ  )rH   rÖ   s   &&r.   Úset_input_embeddingsÚ2InstructBlipVideoQFormerModel.set_input_embeddingsç  s   € Ø*/�‰Ö'r1   c                óL  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
r)   r¢   r×   Nr�   r£   ro  rp  râ   rO   )r?   Ú
LongTensorrn   rQ   r   r   r  r   )r,   r-   s   "€r.   r/   r×  í  s­   ø€ ÷ 9
ñ 9
á×#Ñ#ð9
ñ ×)Ñ)¨DÕ0ð9
ñ ×&Ñ&¨Õ-ð	9
ñ
 —l‘l TÕ)ð9
ñ  %×0Ñ0°4Õ7ð9
ñ !&× 1Ñ 1°DÕ 8ð9
ñ Ñ+Õ,ð9
ñ 
‰u× Ñ Õ	!Ñ$PÕ	Pñ9
r1   c                ó”  € Vf   Vf   \        R4      hVe   VP                  ^,          M^ pV P                  VVVR7      p	\        V P                  V	VR7      pVe1   V^ 8”  d   V	RRV1R3,          MT	p
\        V P                  V
VVR7      pV P
                  ! V	3RVRVR	VR
V/VB pVP                  pVR,          p\        VVR7      # )a  
query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
    Hidden states to be used in the attention computation. If cross-attention,
    will be used for the query (i.e., key and value will use the encoder_hidden_states).
Nz7You have to specify query_embeds when input_ids is None)r¢   r�   r£   )r*   rA  r×   rZ   )r*   rA  r×   ro  r×   ro  rp  r»  rQ  rS  )rî   r\   rL   r
   r*   rJ  r@  r   )rH   r¢   r×   r�   r£   ro  rp  râ   r»  Úembedding_outputÚquery_embedding_outputrT  Úsequence_outputrU  s   &&&&&&&,      r.   r|   Ú%InstructBlipVideoQFormerModel.forwardê  s  € ð$ Ò Ò!5ÜÐVÓWÐWà0<Ò0H�|×)Ñ)¨!Ö,ÈaˆàŸ?™?ØØ%Ø%ð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð "Ò-ð O[Ð]^ÔN^Ð%5°a¸¸,¸ÈÐ6IÖ%JÐdtÐ"Ü%>Ø—{‘{Ø4Ø5Ø&;ô	&Ð"ð ,0¯<ª<Øñ,
à)ð,
ð #8ð,
ð $:ð	,
ð
 &ð,
ð ñ,
ˆð *×;Ñ;ˆØ'¨Õ0ˆä;Ø-Ø'ô
ð 	
r1   )r*   rL   rJ  )NNNNN)r   r€   r�   r‚   r¨   rË   rÍ   rÌ   rÎ   rÅ   r   rÆ   r^  r9   rY  rÞ  r   r   r   r|   rƒ   r„   r…   r†   s   @@r.   rÒ  rÒ  Å  s£   ù‡ € ñð
 #'ÐØ€NØÐØÐð 	Ð6ØÙÐEÈQÐ[gÔhð
ð 	ÙÐEÈQÐ[lÔmð
ðÐ÷ó ò/ò0ð  ØØ÷9
ñ 9
ó ó ó  ÷9
ð 9
r1   rÒ  zV
    Class defining the outputs of [`InstructBlipVideoForConditionalGeneration`].
    )Úcustom_introc                   óV   a € ] tR tRt o RtRtRtRtRtRt	V 3R lR lt
V 3R ltRtV tR# )Ú4InstructBlipVideoForConditionalGenerationModelOutputi)  a~  
loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Language modeling loss from the language model.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head of the language model.
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
    Outputs of the language model.
Nc                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r)   rO   )r  r   )r,   r-   s   "€r.   r/   ÚAInstructBlipVideoForConditionalGenerationModelOutput.__annotate__C  s   ø€ ÷ 
ñ 
™%¡�*ñ 
r1   c                ó    a € \         ;QJ d%    . V 3R  lS P                  4        4       F  NK  	  5# ! V 3R  lS P                  4        4       4      # )c              3   óx   <"  € T F/  pVR9  d
   SV,          M\        SV4      P                  4       x € K1  	  R# 5i)Úvision_outputsN©rï  Úqformer_outputsÚlanguage_model_outputs)ÚgetattrÚto_tuple)Ú.0ÚkrH   s   & €r.   Ú	<genexpr>ÚPInstructBlipVideoForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>D  sC   øé € ð 
ñ !�ð ÐWÔWð �ŽGä˜˜qÓ!×*Ñ*Ó,ô-ó !ùs   ƒ7:)r  ÚkeysrX  s   fr.   rô  Ú=InstructBlipVideoForConditionalGenerationModelOutput.to_tupleC  sE   ø€ ßŒuô 
ð —Y‘Y”[ó	
�uð 	
ˆuô 
ð —Y‘Y”[ó	
ó 
ð 	
r1   c                óô   <€ V ^8„  d   Qh/ S[ S[P                  ,          R,          ;R&   S[ S[P                  ,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[S[,          R,          ;R&   # )r)   NÚlossÚlogitsrï  rñ  rò  )r  r?   rn   r   r   r   r   )r,   r-   s   "€r.   r/   rì  )  s{   ø‡ ‚ ñ( ‘×!Ñ!Õ
" TÕ
)Ñ0ñ) ñ* ‘%×#Ñ#Õ$ tÕ+Ñ2ñ+ ñ, /°Õ5Ñ<ñ- ñ. BÀDÕHÑOñ/ ñ0 3±_ÕDÀtÕKÑRò1 r1   rÀ   )r   r€   r�   r‚   r¨   rü  rý  rï  rñ  rò  rô  rÒ   rƒ   r„   ©r-   s   @r.   ré  ré  )  s7   ø‡ € ñð -1€DØ.2€FØ8<€NØKO€OØNRÐ÷
ð 
÷5 ƒ r1   ré  z`
    InstructBlipVideo base Model consisting of language model, qformer and vision encoder.
    c                   óŠ   a a€ ] tR tRt oRtR.tV3R lV 3R lltR tV3R lR lt]	]
RV3R	 lR
 ll4       4       tRtVtV ;t# )r¶   iL  rm   r¸   c                ó    <€ V ^8„  d   QhRS[ /# r(   r¾   )r,   r-   s   "€r.   r/   Ú#InstructBlipVideoModel.__annotate__U  s   ø€ ÷ ñ Ñ6ñ r1   c                ó  <€ \         SV `  V4       \        VP                  4      V n        \
        P                  ! \        P                  ! ^VP                  VP                  P                  4      4      V n        \        VP                  4      V n        \
        P                  ! VP                  P                  VP                   P                  4      V n        \$        P&                  ! VP                   4      V n        V P+                  4        R# ©r3   N)r8   r9   rE  Úvision_configÚvision_modelr   r>   r?   rõ   Únum_query_tokensÚqformer_configr:   r¸   rÒ  Úqformerrò   Útext_configÚlanguage_projectionr   Úfrom_configÚlanguage_modelrL  rG   s   &&€r.   r9   ÚInstructBlipVideoModel.__init__U  s·   ø€ Ü‰Ñ˜Ô ä8¸×9MÑ9MÓNˆÔÜŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4°V×5JÑ5JÓKˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ Ü'×3Ò3°F×4FÑ4FÓGˆÔð 	�‰Ör1   c                ó.  € V P                   p\        V4      ^8”  d@   RV9  d9   \        P                  P	                  4       ^8”  d   \
        P                  R4       \        V P                  R4      '       d   RV P                  P                  n
        R# R# ©z•
Some pre-processing hacks to make the model `accelerate` compatible. Check
https://github.com/huggingface/transformers/pull/21707 for more details.
r  a   The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models.Ú_hf_hookTN©Úhf_device_mapÚlenr?   ÚcudaÚdevice_countÚloggerÚwarningrb  r  r  Úio_same_device©rH   r  s   & r.   Ú_preprocess_accelerateÚ-InstructBlipVideoModel._preprocess_accelerateb  ó|   € ð
 ×*Ñ*ˆäˆ}Ó Ô!Ð&6¸mÔ&KÔPU×PZÑPZ×PgÑPgÓPiÐlmÔPmä�N‰NðMôô �4×&Ñ&¨
×3Ò3Ø:>ˆD×Ñ×(Ñ(Ö7ñ 4r1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r)   r¢   rA  ©r?   rá  rn   )r,   r-   s   "€r.   r/   r  v  ó'   ø€ ÷ "ñ "©e×.>Ñ.>ð "Éu×O`ÑO`ñ "r1   c           
     ón  € Vfo   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pMWP                  P                  8H  pVP                  R4      P                  VP                  4      pV# ©zJ
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
©rq   r¡   rÀ   r[   )
rY  r?   Úfullr*   Úimage_token_idÚlongr¡   ÚallÚ	unsqueezers   ©rH   r¢   rA  Úspecial_image_masks   &&& r.   Úget_placeholder_maskÚ+InstructBlipVideoModel.get_placeholder_maskv  ó˜   € ð ÒØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¯k©k×.HÑ.HÑ!HÐà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ!Ð!r1   c                ó„  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[RS[R,          RS[S[,          RS[S[,          /# )r)   rm   Úqformer_input_idsÚqformer_attention_maskNr¢   r×   Údecoder_input_idsÚdecoder_attention_maskrA  rj   Ú	use_cacherâ   rO   )	r?   rn   rá  rQ   ro   r   r   r  ré  )r,   r-   s   "€r.   r/   r  ‡  sí   ø€ ÷ q
ñ q
á×'Ñ'ðq
ñ !×,Ñ,ðq
ñ !&× 0Ñ 0°4Õ 7ð	q
ñ
 ×$Ñ$ tÕ+ðq
ñ ×(Ñ(¨4Õ/ðq
ñ !×+Ñ+¨dÕ2ðq
ñ !&× 0Ñ 0°4Õ 7ðq
ñ —|‘| dÕ*ðq
ñ #'ðq
ñ ˜$•;ðq
ñ Ñ+Õ,ðq
ñ 
ÑEÕ	Eñq
r1   c                óæ  € VP                   w  rÍrïpVP                  WÍ,          WïV4      pV P                  ! RRVRV	/VB pV^ ,          p\        P                  ! VP                  4       RR \        P                  VP                  R7      pV P                  P                  VP                   ^ ,          RR4      p\        P                  ! VP                  4       RR \        P                  VP                  R7      pVf   \        P                  ! V4      pVP                  V^ R7      pVP                  V^ R7      pVP                  VP                  4      p\        P                  ! VV.^R7      pV P                  ! RRVRVRVR	VR
V/VB pV^ ,          RRVP                  ^4      1R3,          pV P                  V4      pVP                  WÀP                   P"                  V,          R4      pVfT   V P$                  P'                  4       ! V4      pW@P                   P(                  8H  pVf   \        P                  ! V4      pMmW€P'                  4       ! \        P*                  ! RV P                   P(                  \        P                  VP                  R7      4      8H  pVP-                  R4      pVP/                  R4      P                  VP                  4      pVP                  VP                  VP0                  4      pVP3                  VV4      pV P                   P4                  '       d   V P$                  ! RRVRVRV
/VB pMV P$                  ! RRVRVRVRVRV
/VB p\7        VVVR7      # )aU  
qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
    to serve as text prompt, which the Q-Former model will encode.

    Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for
    details.

    [What are input IDs?](../glossary#input-ids)
qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    Only relevant in case an encoder-decoder language model (like T5) is used.
rm   rj   Nr#  rX   r¢   r×   r£   ro  rp  rZ   rA  r3  r1  r2  rð  rÀ   r[   )r\   r_   r  r?   ÚonesrU   r&  r¡   r¸   rv   Ú	ones_likeÚrepeat_interleavers   rd   r  r
  r*   r  r  rY  Úvideo_token_idr$  r'  r(  rq   Úmasked_scatterÚuse_decoder_only_language_modelré  )rH   rm   r/  r0  r¢   r×   r1  r2  rA  rj   r3  râ   rw   ÚframesÚchannelrM   rN   rï  Úimage_embedsÚimage_attention_maskr¸   Úquery_attention_maskÚquery_outputsÚquery_outputÚlanguage_model_inputsr*  Úoutputss   &&&&&&&&&&&,               r.   r|   ÚInstructBlipVideoModel.forward…  s�  € ðP 6B×5GÑ5GÑ2ˆ
˜G¨UØ#×+Ñ+¨JÕ,?ÀÐRWÓXˆà×*Ò*ñ 
Ø%ð
à%=ð
ð ñ
ˆð
 & aÕ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐà!Ò)Ü%*§_¢_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÓNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÓ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@ÐBXÐ+YÐ_`Ô!aÐØŸšñ 
Ø'ð
à1ð
ð &ð
ð #/ð	
ð
 $8ð
ð ñ
ˆð % QÕ'¨Ð+A¨\×->Ñ->¸qÓ-AÐ+AÀ1Ð(DÕEˆð !%× 8Ñ 8¸Ó FÐð !6× =Ñ =¸jÏ+É+×JfÑJfÐioÕJoÐqsÓ tÐØÒ Ø ×/Ñ/×DÑDÔFÀyÓQˆMØ!*¯k©k×.HÑ.HÑ!HÐØÒ%Ü!&§¢°Ó!;�øà!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ðà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ð6Ø×)Ò)ñ Ø+ðà-ðð $ðð ñ	‰Gð ×)Ò)ñ Ø+ðà-ðð #4ðð (>ð	ð
 $ðð ñˆGô DØ)Ø)Ø#*ô
ð 	
r1   ©r  r
  r  r¸   r  )NNNNNNFN)r   r€   r�   r‚   r]  Ú_keep_in_fp32_modulesr9   r  r+  r   r   r|   rƒ   r„   r…   r†   s   @@r.   r¶   r¶   L  sX   ù‡ € ð %€OØ+Ð,Ð÷ó ò?÷("ð "ð Ø÷q
ñ q
ó ó ÷q
ð q
r1   r¶   c                   ó8   a € ] tR tRt o RtRtRtV 3R ltRtV t	R# )Ú'BaseModelOutputWithVisionQformerOutputsiû  zÉ
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
Nc                óN   <€ V ^8„  d   Qh/ S[ R,          ;R&   S[R,          ;R&   # )r)   Nrï  rñ  )r   r   )r,   r-   s   "€r.   r/   Ú4BaseModelOutputWithVisionQformerOutputs.__annotate__û  s*   ø‡ ‚ ñ /°Õ5Ñ<ñ ñ BÀDÕHÑOò r1   rÀ   )
r   r€   r�   r‚   r¨   rï  rñ  rÒ   rƒ   r„   rþ  s   @r.   rH  rH  û  s   ø‡ € ñð 9=€NØKO€O÷ ƒ r1   rH  a¯  
    InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                   ó0  a a€ ] tR tRt oRtRtR.tV3R lV 3R lltR tV3R lR	 lt	RV 3R
 llt
R tR tV3R lR lt]]RV3R lR ll4       4       t]P$                  ! 4       RV3R lR ll4       t]]RV3R lR ll4       4       tV3R ltRtVtV ;t# )rµ   i	  rm   Tr¸   c                ó    <€ V ^8„  d   QhRS[ /# r(   r¾   )r,   r-   s   "€r.   r/   Ú6InstructBlipVideoForConditionalGeneration.__annotate__  s   ø€ ÷ ñ Ñ6ñ r1   c                ó®  <€ \         SV `  V4       \        P                  VP                  4      V n        \        P                  ! \        P                  ! ^VP                  VP                  P                  4      4      V n        \        P                  VP                  4      V n        \        P                   ! VP                  P                  VP"                  P                  4      V n        VP&                  '       d"   \(        P*                  ! VP"                  4      pM \,        P*                  ! VP"                  4      pW n        V P1                  4        R# r  )r8   r9   rE  Ú_from_configr  r  r   r>   r?   rõ   r  r  r:   r¸   rÒ  r  rò   r	  r
  r:  r    r  r!   r  rL  )rH   r*   r  rI   s   && €r.   r9   Ú2InstructBlipVideoForConditionalGeneration.__init__  sè   ø€ Ü‰Ñ˜Ô ä8×EÑEÀf×FZÑFZÓ[ˆÔäŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4×AÑAÀ&×BWÑBWÓXˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ à×1×1Ð1Ü1×=Ò=¸f×>PÑ>PÓQ‰Nä2×>Ò>¸v×?QÑ?QÓRˆNà,Ôð 	�‰Ör1   c                ó<   € V P                   P                  V4       R # r  )r  Úset_output_embeddings)rH   Únew_embeddingss   &&r.   rR  Ú?InstructBlipVideoForConditionalGeneration.set_output_embeddings-  s   € Ø×Ñ×1Ñ1°.ÖAr1   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rë  )r   rÙ   )r,   r-   s   "€r.   r/   rM  0  s   ø€ ÷ ;ñ ;¡r§y¡yñ ;r1   c                ó6   € V P                   P                  4       # r  )r  Úget_output_embeddingsrX  s   &r.   rW  Ú?InstructBlipVideoForConditionalGeneration.get_output_embeddings0  s   € Ø×"Ñ"×8Ñ8Ó:Ð:r1   c                ó`   <€ Vf   V P                   P                  4       # \        SV `  VR7      # )N)Úmodality)r  Úget_encoderr8   )rH   rZ  rI   s   &&€r.   r[  Ú5InstructBlipVideoForConditionalGeneration.get_encoder3  s1   ø€ ØÒØ×&Ñ&×2Ñ2Ó4Ð4ä‘7Ñ&°Ð&Ó9Ð9r1   c                ó6   € V P                   P                  4       # r  )r  Úget_decoderrX  s   &r.   r^  Ú5InstructBlipVideoForConditionalGeneration.get_decoder9  s   € Ø×"Ñ"×.Ñ.Ó0Ð0r1   c                ó.  € V P                   p\        V4      ^8”  d@   RV9  d9   \        P                  P	                  4       ^8”  d   \
        P                  R4       \        V P                  R4      '       d   RV P                  P                  n
        R# R# r  r  r  s   & r.   r  Ú@InstructBlipVideoForConditionalGeneration._preprocess_accelerate<  r  r1   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  r  )r,   r-   s   "€r.   r/   rM  P  r   r1   c           
     ón  € Vfo   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pMWP                  P                  8H  pVP                  R4      P                  VP                  4      pV# r"  )
rY  r?   r$  r*   r8  r&  r¡   r'  r(  rs   r)  s   &&& r.   r+  Ú>InstructBlipVideoForConditionalGeneration.get_placeholder_maskP  r-  r1   c                ó¬  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[RS[R,          RS[S[,          RS[S[,          /# )r)   rm   r/  r0  Nr¢   r×   r1  r2  rA  Úlabelsrj   r3  râ   rO   )r?   rn   rá  ro   r   r   r  ré  )r,   r-   s   "€r.   r/   rM  a  s  ø€ ÷ D
ñ D
á×'Ñ'ðD
ñ !×,Ñ,ðD
ñ !&× 0Ñ 0°4Õ 7ð	D
ñ
 ×$Ñ$ tÕ+ðD
ñ ×(Ñ(¨4Õ/ðD
ñ !×+Ñ+¨dÕ2ðD
ñ !&× 0Ñ 0°4Õ 7ðD
ñ ×(Ñ(¨4Õ/ðD
ñ × Ñ  4Õ'ðD
ñ #'ðD
ñ ˜$•;ðD
ñ Ñ+Õ,ðD
ñ 
ÑEÕ	EñD
r1   c                óø  € V P                   ! V3RVRVRV
/VB pVP                  pVP                  pVP                  pVf   V P	                  4       ! V4      pVf   \
        P                  ! V4      pVP                  VP                  VP                  4      pV P                  WHR7      pVP                  VV4      pV P                  P                  '       d_   V P                  ! RRVRVRV/VB pV^ ,          pRpV	e7   V P                  ! RR	VR
V	RV P                  P                   P"                  /VB pM6V P                  ! RRVRVRVRVR
V	RV/VB pVP$                  pVP&                  p\)        VVVVVR7      # )aÐ	  
qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
    The sequence used as a prompt to be fed to the Q-Former module.
qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
    Mask to avoid performing attention on padding token indices.

Examples:

```python
>>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration
>>> import torch
>>> from huggingface_hub import hf_hub_download
>>> import av
>>> import numpy as np

>>> def read_video_pyav(container, indices):
...     '''
...     Decode the video with PyAV decoder.
...     Args:
...         container (`av.container.input.InputContainer`): PyAV container.
...         indices (`list[int]`): List of frame indices to decode.
...     Returns:
...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
...     '''
...     frames = []
...     container.seek(0)
...     start_index = indices[0]
...     end_index = indices[-1]
...     for i, frame in enumerate(container.decode(video=0)):
...         if i > end_index:
...             break
...         if i >= start_index and i in indices:
...             frames.append(frame)
...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])

>>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto")
>>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

>>> file_path = hf_hub_download(
...       repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
... )
>>> container = av.open(file_path)

>>> # sample uniformly 4 frames from the video
>>> total_frames = container.streams.video[0].frames
>>> indices = np.arange(0, total_frames, total_frames / 4).astype(int)
>>> clip = read_video_pyav(container, indices)

>>> prompt = "What is happening in the video?"
>>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device)

>>> outputs = model.generate(
...     **inputs,
...     do_sample=False,
...     num_beams=5,
...     max_length=256,
...     repetition_penalty=1.5,
...     length_penalty=1.0,
... )
>>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
>>> print(generated_text)
"A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front"
```r/  r0  rj   N©rA  rA  r×   r3  rý  rf  r�   r1  r2  )rü  rý  rï  rñ  rò  rÀ   )Úget_video_featuresrR  rñ  rï  rY  r?   r6  rs   r¡   rq   r+  r9  r*   r:  r  Úloss_functionr	  r�   rü  rý  ré  )rH   rm   r/  r0  r¢   r×   r1  r2  rA  rf  rj   r3  râ   Úvideo_featuresrB  rñ  rï  r*  rC  rý  rü  s   &&&&&&&&&&&&,        r.   r|   Ú1InstructBlipVideoForConditionalGeneration.forward_  sü  € ð` CG×BYÒBYØñC
à/ðC
ð $:ðC
ð &>ð	C
ð
 ñC
ˆð !/× <Ñ <ÐØ(×8Ñ8ˆØ'×6Ñ6ˆàÒ Ø ×5Ñ5Ô7¸	ÓBˆMàÒ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ó^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ð6Ø×)Ò)ñ Ø+ðà-ðð $ðð ñ	ˆGð ˜Q•ZˆFØˆDØÒ!Ø×)Ò)ñ Ø!ðØ*0ðØ=A¿[¹[×=TÑ=T×=_Ñ=_ðØciñ�øð
 ×)Ò)ñ Ø+ðà-ðð #4ðð (>ð	ð
 ðð $ðð ñˆGð —<‘<ˆDØ—^‘^ˆFäCØØØ)Ø+Ø#*ô
ð 	
r1   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R	S[ P                  /# )
r)   rm   r/  Nr0  r¢   r×   rA  rj   rO   )r?   rn   rá  ro   )r,   r-   s   "€r.   r/   rM  è  s¨   ø€ ÷ Cñ Cá×'Ñ'ðCñ !×+Ñ+¨dÕ2ðCñ !&× 0Ñ 0°4Õ 7ð	Cñ
 ×#Ñ# dÕ*ðCñ ×(Ñ(¨4Õ/ðCñ ×(Ñ(¨4Õ/ðCñ #'ðCñ 
×	Ñ	ñCr1   c                ó˜  € \        V R4      '       d   V P                  4        VP                  ^ ,          p	V P                  VVVVR7      p
V
P                  pVf¿   Vf¥   V P
                  P                  .V P
                  P                  ,          ^,          pWÀP
                  P                  P                  .,           p\        P                  ! V.\        P                  VP                  R7      pVP                  V	^4      pV P                  4       ! V4      pVf   \        P                   ! V4      pVP#                  VP                  VP$                  4      pV P'                  WFR7      pVP)                  Wë4      pRVRV/pV P*                  P
                  P,                  '       g   WOR&   V P*                  P.                  ! R/ VBVB pV# )	aA  
Overrides `generate` function to be able to use the model as a conditional generator.

Args:
    pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or
        (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed.
    qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt to be fed to the Q-Former module.
    qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt for the generation.
    attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Embedded representation of the inputs. Should be float, not int tokens.
    interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
        Whether to interpolate the positional encoding of the image embeddings.

Returns:
    captions (list): A list of strings of length batch_size * num_captions.
r  )r/  r0  rj   r#  rh  rA  r×   r¢   rÀ   )rb  r  r\   ri  rR  r*   Úvideo_token_indexr  r	  Úbos_token_idr?   rü   r&  r¡   ÚrepeatrY  r6  rs   rq   r+  r9  r  Úis_encoder_decoderÚgenerate)rH   rm   r/  r0  r¢   r×   rA  rj   Úgenerate_kwargsrw   rk  rB  Úvideo_tokensÚstart_tokensr*  ÚinputsrC  s   &&&&&&&&,        r.   rs  Ú2InstructBlipVideoForConditionalGeneration.generateç  sœ  € ôD �4˜×)Ò)à×'Ñ'Ô)à!×'Ñ'¨Õ*ˆ
ØBF×BYÑBYØØ/Ø#9Ø%=ð	 CZó C
ˆð !/× <Ñ <ÐàÒ ØÒ Ø $§¡× =Ñ =Ð>ÀÇÁ×A]ÑA]Õ]Ð`aÕa�Ø+¯{©{×/FÑ/F×/SÑ/SÐ.TÕT�Ü!ŸLšL¨,¨¼u¿z¹zÐR^×ReÑReÔf�	Ø%×,Ñ,¨Z¸Ó;�	Ø ×5Ñ5Ô7¸	ÓBˆMàÒ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ó^ÐØ%×4Ñ4Ð5GÓ_ˆà! =Ð2BÀNÐSˆØ×"Ñ"×)Ñ)×<×<Ð<Ø"+�;Ñà×%Ñ%×.Ò.ÑK°ÐK¸?ÑKˆàˆr1   c                ó¶   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[,          /# )r)   rm   r/  r0  Nrj   râ   rO   )r?   rn   rá  ro   r   r   r  rH  )r,   r-   s   "€r.   r/   rM  /  sv   ø€ ÷ Dñ Dá×'Ñ'ðDñ !×+Ñ+ðDñ !&× 0Ñ 0°4Õ 7ð	Dñ
 #'¨¥+ðDñ Ñ+Õ,ðDñ 
Ñ8Õ	8ñDr1   c                ól  € VP                   w  rgr‰p
VP                  Wg,          W‰V
4      pV P                  ! RRVRV/VB p\        VP                  VP
                  VP                  VP                  VRR7      pV^ ,          p\        P                  ! VP                  4       RR \        P                  VP                  R7      pV P                  P                  VP                   ^ ,          RR4      p\        P                  ! VP                  4       RR \        P                  VP                  R7      pVf   \        P                  ! V4      pVP!                  V^ R7      pVP!                  V^ R7      pVP#                  VP                  4      p\        P$                  ! Wó.^R7      pV P&                  ! RRVRVR	VR
VRV/VB pVVn        V^ ,          RRVP                  ^4      1R3,          pV P+                  V4      pVP                  W`P,                  P.                  V,          R4      pVVn        V# )aÊ  
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
    The tensors corresponding to the input images.
qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
    The sequence used as a prompt to be fed to the Q-Former module.
qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
    Mask to avoid performing attention on padding token indices.
rm   rj   N)r@  rR  r  rF  rï  rñ  r#  rX   r¢   r×   r£   ro  rp  rZ   rÀ   r[   )r\   r_   r  rH  r@  rR  r  rF  r?   r5  rU   r&  r¡   r¸   rv   r6  r7  rs   rd   r  rñ  r
  r*   r  )rH   rm   r/  r0  rj   râ   rw   r;  r<  rM   rN   rï  r=  r>  r¸   r?  rñ  rA  rk  s   &&&&&,             r.   ri  Ú<InstructBlipVideoForConditionalGeneration.get_video_features-  sN  € ð( 6B×5GÑ5GÑ2ˆ
˜G¨UØ#×+Ñ+¨JÕ,?ÀÐRWÓXˆà59×5FÒ5Fñ 6
Ø%ð6
à%=ð6
ð ñ6
ˆô
 AØ,×>Ñ>Ø(×6Ñ6Ø(×6Ñ6Ø%×0Ñ0Ø)Ø ô
ˆð & aÕ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐà!Ò)Ü%*§_¢_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÓNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÓ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@Ð+YÐ_`Ô!aÐØŸ,š,ñ 
Ø'ð
à1ð
ð &ð
ð #/ð	
ð
 $8ð
ð ñ
ˆð *9ˆÔ&Ø& qÕ)¨!Ð-C¨|×/@Ñ/@ÀÓ/CÐ-CÀQÐ*FÕGˆð ×1Ñ1°,Ó?ˆð (×/Ñ/°
¿K¹K×<XÑ<XÐ[aÕ<aÐceÓfˆØ'5ˆÔ$àÐr1   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r(   r¾   )r,   r-   s   "€r.   r/   rM  	  s   ø‡ ‚ ñ $Ñ#ò r1   rE  r  )	NNNNNNNFN)NNNNNFr\  )r   r€   r�   r‚   r]  rÏ   rF  r9   rR  rW  r[  r^  r  r+  r   r   r|   r?   rÑ   rs  ri  rÒ   rƒ   r„   r…   r†   s   @@r.   rµ   rµ   	  sÂ   ù‡ € ð %€Oà!ÐØ+Ð,Ð÷ó ò(B÷;ð ;÷:ò1ò?÷("ð "ð Ø÷D
ñ D
ó ó ðD
ðL ‡]‚]ƒ_÷Cñ Có ðCðJ Ø÷Dñ Dó ó ðD÷M	 … r1   rµ   )r­   )rE  rª   rÒ  r¶   rµ   )OÚcollections.abcr   Údataclassesr   Útypingr   r?   r   Ú r   r³   Úactivationsr   Ú
generationr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úautor   r    r!   Úconfiguration_instructblipvideor"   r#   r$   Ú
get_loggerr   r  rÙ   r&   rˆ   rª   rå   rÃ   r  rÄ   r1  rE  rÆ   rÇ   r‹  r—  r¢  rÅ   rÇ  rÒ  ré  r¶   rH  rµ   Ú__all__rÀ   r1   r.   Ú<module>r�     s;  ðõ, %Ý !Ý ã Ý å &Ý !Ý )Ý 6Ý 9÷÷ ÷ GÝ &Ý 6ß j× jÝ 7ß Eß IÑ I÷ñ ð 
×	Ò	˜HÓ	%€ôG¨¯	©	ô GôT-¨¯©ô -ð` ô i ó  ió ð i÷H%ô.G) §¡ô G)ôT˜2Ÿ9™9ô ôÐ$>ô ô@@˜rŸy™yô @ô@3Ð#Cô 3ôlE)°·±ô E)ôP¨¯©ô ô ¨¯	©	ô  ô2¨2¯9©9ô ô R§Y¡Yô ôRÐ$>ô Rôj!
 b§i¡iô !
ôHa
Ð$Dô a
ñH ðôð
 ô
¸;ó 
ó óð
ñ: ðôô
g
Ð=ó g
óð
g
ðT Ø
ô	PÐ.Hó 	Pó ó ð	Pñ ðôôaÐ0PÐRaó aóðaòH‚r1   