+
    œ°³j¼ç  ã                   óx  € R t ^ RIHt ^ RIHt ^ RIHt ^ RIt^ RIHt ^RI	H
t ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHt ^RIHtHtHtHtHtHt ^RIHtHt ^RI H!t! ^RI"H#t# ^RI$H%t%H&t&H't'H(t(H)t)H*t* ^RI+H,t, ^RI-H.t.H/t/ ^RI0H1t1H2t2H3t3 ^RI4H5t5H6t6H7t7 ])Pp                  ! ]94      t:]'] ! R R]4      4       4       t;]'! RR7      ] ! R R]%4      4       4       t< ! R R]Pz                  4      t>RCR R llt? ! R R ]Pz                  4      t@ ! R! R"]Pz                  4      tA ! R# R$]4      tB]' ! R% R&]4      4       tC ! R' R(]Pz                  4      tD ! R) R*]C4      tE ! R+ R,]Pz                  4      tF ! R- R.]Pz                  4      tG ! R/ R0]Pz                  4      tH ! R1 R2]Pz                  4      tI ! R3 R4]Pz                  4      tJ ! R5 R6]4      tK ! R7 R8]Pz                  4      tL ! R9 R:]Pz                  4      tM ! R; R<]C4      tN]'! R=R7       ! R> R?]C4      4       tO]'! R@R7       ! RA RB]C]4      4       tP. RDOtQR# )EzPyTorch InstructBLIP model.)ÚCallable)Ú	dataclass)ÚAnyN)Únn)Úinitialization)ÚACT2FN)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)Ú	AutoModelÚAutoModelForCausalLMÚAutoModelForSeq2SeqLM)ÚInstructBlipConfigÚInstructBlipQFormerConfigÚInstructBlipVisionConfigc                   ó8   a € ] tR t^2t o RtRtRtV 3R ltRtV t	R# )Ú'BaseModelOutputWithVisionQformerOutputszÉ
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
Nc                óN   <€ V ^8„  d   Qh/ S[ R,          ;R&   S[R,          ;R&   # )é   NÚvision_outputsÚqformer_outputs)r   r   )ÚformatÚ__classdict__s   "€Úw/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/instructblip/modeling_instructblip.pyÚ__annotate__Ú4BaseModelOutputWithVisionQformerOutputs.__annotate__2   s*   ø‡ ‚ ñ /°Õ5Ñ<ñ ñ BÀDÕHÑOò ó    © )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r)   r*   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r,   s   @r-   r&   r&   2   s   ø‡ € ñð 9=€NØKO€O÷ ƒ r0   r&   zQ
    Class defining the outputs of [`InstructBlipForConditionalGeneration`].
    )Úcustom_introc                   óV   a € ] tR t^@t o RtRtRtRtRtRt	V 3R lR lt
V 3R ltRtV tR# )Ú/InstructBlipForConditionalGenerationModelOutputa~  
loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Language modeling loss from the language model.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head of the language model.
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
    Outputs of the language model.
Nc                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r(   Úreturn)Útupler   )r+   r,   s   "€r-   r.   Ú<InstructBlipForConditionalGenerationModelOutput.__annotate__[   s   ø€ ÷ 
ñ 
™%¡�*ñ 
r0   c                ó    a € \         ;QJ d%    . V 3R  lS P                  4        4       F  NK  	  5# ! V 3R  lS P                  4        4       4      # )c              3   óx   <"  € T F/  pVR9  d
   SV,          M\        SV4      P                  4       x € K1  	  R# 5i)r)   N©r)   r*   Úlanguage_model_outputs)ÚgetattrÚto_tuple)Ú.0ÚkÚselfs   & €r-   Ú	<genexpr>ÚKInstructBlipForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>\   sC   øé € ð 
ñ !�ð ÐWÔWð �ŽGä˜˜qÓ!×*Ñ*Ó,ô-ó !ùs   ƒ7:)rA   Úkeys©rK   s   fr-   rH   Ú8InstructBlipForConditionalGenerationModelOutput.to_tuple[   sE   ø€ ßŒuô 
ð —Y‘Y”[ó	
�uð 	
ˆuô 
ð —Y‘Y”[ó	
ó 
ð 	
r0   c                óô   <€ V ^8„  d   Qh/ S[ S[P                  ,          R,          ;R&   S[ S[P                  ,          R,          ;R&   S[R,          ;R&   S[R,          ;R&   S[S[,          R,          ;R&   # )r(   NÚlossÚlogitsr)   r*   rF   )rA   ÚtorchÚFloatTensorr   r   r   r   )r+   r,   s   "€r-   r.   rB   @   s{   ø‡ ‚ ñ* ‘×!Ñ!Õ
" TÕ
)Ñ0ñ+ ñ, ‘%×#Ñ#Õ$ tÕ+Ñ2ñ- ñ. /°Õ5Ñ<ñ/ ñ0 BÀDÕHÑOñ1 ñ2 3±_ÕDÀtÕKÑRò3 r0   r1   )r2   r3   r4   r5   r6   rR   rS   r)   r*   rF   rH   r7   r8   r9   r:   s   @r-   r=   r=   @   s7   ø‡ € ñð -1€DØ.2€FØ8<€NØKO€OØNRÐ÷
ð 
÷7 ƒ r0   r=   c                   óf   a a€ ] tR t^et oV3R lV 3R lltV3R lR ltRV3R lR lltRtVtV ;t	# )	ÚInstructBlipVisionEmbeddingsc                ó    <€ V ^8„  d   QhRS[ /# ©r(   Úconfig©r$   )r+   r,   s   "€r-   r.   Ú)InstructBlipVisionEmbeddings.__annotate__f   s   ø€ ÷ cñ cÑ7ñ cr0   c                ó’  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        \        P                  ! \        P                  ! ^^V P                  4      4      V n        \        P                  ! ^V P                  V P                  V P                  R7      V n        V P
                  V P                  ,          ^,          V n        V P                  ^,           V n        \        P                  ! \        P                  ! ^V P                  V P                  4      4      V n        R# )é   )Úin_channelsÚout_channelsÚkernel_sizeÚstrideN)ÚsuperÚ__init__rZ   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	ParameterrT   ÚrandnÚclass_embeddingÚConv2dÚpatch_embeddingÚnum_patchesÚnum_positionsÚposition_embedding©rK   rZ   Ú	__class__s   &&€r-   rd   Ú%InstructBlipVisionEmbeddings.__init__f   sß   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|š|¬E¯KªK¸¸1¸d¿n¹nÓ,MÓNˆÔä!ŸyšyØ¨¯©ÀDÇOÁOÐ\`×\kÑ\kô 
ˆÔð !ŸO™O¨t¯©Õ>À1ÕDˆÔØ!×-Ñ-°Õ1ˆÔä"$§,¢,¬u¯{ª{¸1¸d×>PÑ>PÐRV×R`ÑR`Ó/aÓ"bˆÖr0   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  /# )r(   Ú
embeddingsÚheightÚwidthr@   ©rT   ÚTensorÚint)r+   r,   s   "€r-   r.   r\   x   s;   ø€ ÷ &Dñ &D±5·<±<ð &DÉð &DÑUXð &DÑ]b×]iÑ]iñ &Dr0   c                óæ  € VP                   ^,          ^,
          pV P                  P                   ^,          ^,
          p\        P                  P	                  4       '       g   WE8X  d   W#8X  d   V P                  # V P                  R,          pV P                  R,          pVP                   R,          pW P
                  ,          p	W0P
                  ,          p
\        VR,          4      pVP                  ^W»V4      pVP                  ^ ^^^4      p\        P                  P                  VWš3RRR7      pVP                  ^ ^^^4      P                  ^RV4      p\        P                  ! Wg3^R7      # )	aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
g      à?ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)ºNNN:Nr^   N)r‚   :r^   NNéÿÿÿÿ)Úshaperp   rT   ÚjitÚ
is_tracingrh   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)rK   ru   rv   rw   rn   ro   Úclass_pos_embedÚpatch_pos_embedr�   Ú
new_heightÚ	new_widthÚsqrt_num_positionss   &&&&        r-   Úinterpolate_pos_encodingÚ5InstructBlipVisionEmbeddings.interpolate_pos_encodingx   sE  € ð !×&Ñ& qÕ)¨AÕ-ˆØ×/Ñ/×5Ñ5°aÕ8¸1Õ<ˆô �y‰y×#Ñ#×%Ò%¨+Ô*FÈ6Ì?Ø×*Ñ*Ð*à×1Ñ1°%Õ8ˆØ×1Ñ1°%Õ8ˆà×Ñ˜rÕ"ˆàŸ™Õ.ˆ
ØŸ_™_Õ,ˆ	ä& }°cÕ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÔCÐCr0   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r(   Úpixel_valuesr’   r@   )rT   rU   Úboolry   )r+   r,   s   "€r-   r.   r\       s0   ø€ ÷ ñ ¡E×$5Ñ$5ð ÑQUð Ñbg×bnÑbnñ r0   c                ó$  € VP                   w  r4rVV P                  P                  P                  pV P                  VP	                  VR 7      4      pVP                  ^4      P                  ^^4      pV P                  P                  V^R4      P	                  V4      p	\        P                  ! W˜.^R7      p
V'       d   V P                  W¥V4      pMV P                  pW«RRV
P                  ^4      1R3,          P	                  V4      ,           p
V
# ))Údtyper€   r‚   Nrƒ   )r„   rm   Úweightr˜   ÚtoÚflattenÚ	transposerk   ÚexpandrT   rŒ   r’   rp   r}   )rK   r•   r’   Ú
batch_sizeÚ_rv   rw   Útarget_dtypeÚpatch_embedsÚclass_embedsru   rp   s   &&&         r-   ÚforwardÚ$InstructBlipVisionEmbeddings.forward    sñ   € Ø'3×'9Ñ'9Ñ$ˆ
�vØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆØ×+Ñ+×2Ñ2°:¸qÀ"ÓE×HÑHÈÓVˆÜ—Y’Y Ð;ÀÔCˆ
ß#Ø!%×!>Ñ!>¸zÐSXÓ!YÑà!%×!8Ñ!8ÐØ°QÐ8L¸*¿/¹/È!Ó:LÐ8LÈaÐ5OÕ"P×"SÑ"SÐT`Ó"aÕaˆ
ØÐr0   )	rk   rZ   rf   rg   rn   ro   rm   rh   rp   ©F)
r2   r3   r4   r5   rd   r’   r£   r8   r9   Ú__classcell__©rr   r,   s   @@r-   rW   rW   e   s-   ù‡ € ÷có c÷$&Dð &D÷P÷ ò r0   rW   c                óÞ   € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        /# )	r(   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropout)r   ÚModulerT   ry   Úfloat)r+   s   "r-   r.   r.   °   sg   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ñ%r0   c                 ó~  € \         P                  ! WP                  RR4      4      V,          pVe	   W„,           p\        P                  P                  VRR7      p\        P                  P                  W†V P                  R7      p\         P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )r^   r€   )ÚpÚtrainingrƒ   éþÿÿÿ)	rT   Úmatmulrœ   r   r‰   Úsoftmaxr¯   r´   Ú
contiguous)
r©   rª   r«   r¬   r­   r®   r¯   ÚkwargsÚattn_weightsÚattn_outputs
   &&&&&&&,  r-   Úeager_attention_forwardr¼   °   s˜   € ô —<’< §}¡}°R¸Ó'<Ó=ÀÕG€LØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r0   c                   óZ   a a€ ] tR t^Èt oRtV 3R ltV3R lR ltV3R lR ltRtVt	V ;t
# )ÚInstructBlipAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                ó<  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  V P                  ,          V P                  8w  d'   \        R V P                   RV P                   R24      hV P                  R,          V n	        RV n
        VP                  V n        \        P                  ! V P                  ^V P                  ,          RR7      V n        VP                  '       dj   \        P                   ! \"        P$                  ! V P                  4      4      p\        P                   ! \"        P$                  ! V P                  4      4      pMRpRpVeU   \"        P&                  ! V\"        P(                  ! VRR7      V34      p\        P                   ! V4      V P                  n        \        P                  ! V P                  V P                  4      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).F)ÚbiasN)Úrequires_gradç      à¿)rc   rd   rZ   re   rf   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚ	is_causalÚattention_dropoutr   ÚLinearÚqkvÚqkv_biasri   rT   ÚzerosrŒ   Ú
zeros_likerÀ   Ú
projection)rK   rZ   Úq_biasÚv_biasrÌ   rr   s   &&   €r-   rd   ÚInstructBlipAttention.__init__Ë   su  ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Õ8ˆŒØ�=‰=˜4Ÿ>™>Õ)¨T¯^©^Ô;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÕ(ˆŒ
ØˆŒØ!'×!9Ñ!9ˆÔô —9’9˜TŸ^™^¨Q°·±Õ-?ÀeÔLˆŒà�?�?ˆ?Ü—\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆFÜ—\’\¤%§+¢+¨d¯n©nÓ"=Ó>‰FàˆFØˆFàÒÜ—y’y &¬%×*:Ò*:¸6ÐQVÔ*WÐY_Ð!`ÓaˆHÜŸLšL¨Ó2ˆD�H‰HŒMäŸ)š) D§N¡N°D·N±NÓCˆŽr0   c                ó@   <€ V ^8„  d   QhRS[ P                  RS[RS[/# )r(   ÚtensorÚseq_lenÚbszrx   )r+   r,   s   "€r-   r.   Ú"InstructBlipAttention.__annotate__ê   s+   ø€ ÷ eñ e™UŸ\™\ð e±Cð e¹cñ er0   c                óŒ   € VP                  W2V P                  V P                  4      P                  ^^4      P	                  4       # )r^   )r‹   rÄ   rÅ   rœ   r¸   )rK   rÔ   rÕ   rÖ   s   &&&&r-   Ú_shapeÚInstructBlipAttention._shapeê   s5   € Ø�{‰{˜3¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr0   c                ó¼   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  R,          S[S[ P                  ,          R,          3,          /# )r(   Úhidden_statesr@   N)rT   ry   rA   )r+   r,   s   "€r-   r.   r×   í   sJ   ø€ ÷ ")ñ ")á—|‘|ð")ñ 
‰u�|‰|™UŸ\™\¨DÕ0±%¹¿¹Õ2EÈÕ2LÐLÕ	Mñ	")r0   c           	     ó>  € VP                  4       w  r4pV P                  V4      pVP                  W4^V P                  WPP                  ,          4      P	                  ^^ ^^^4      pV^ ,          V^,          V^,          r˜p\
        P                  ! V P                  P                  \        4      p
T
! V VVV	3RRRV P                  '       g   RMV P                  RV P                  /VB w  r¼VP                  W4R4      P                  4       pV P                  V4      pW¼3# )z#Input shape: Batch x Time x Channelr­   Nr¯   ç        r®   rƒ   )r}   rË   r‡   rÄ   rˆ   r   Úget_interfacerZ   Ú_attn_implementationr¼   r´   rÉ   rÇ   r¸   rÏ   )rK   rÜ   r¹   rÖ   Útgt_lenrf   Ú	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer»   rº   s   &&,          r-   r£   ÚInstructBlipAttention.forwardí   s$  € ð #0×"4Ñ"4Ó"6Ñˆ�ià—H‘H˜]Ó+ˆ	à×%Ñ% c°A°t·~±~Ày×TbÑTbÕGbÓc×kÑkØˆq�!�Q˜ó
ˆ	ð 2;¸1µ¸yÈ½|ÈYÐWXÍ\ ,ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØñ		%
ð
  ð	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —J‘Jð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#¸Ó;×FÑFÓHˆØ—o‘o kÓ2ˆàÐ(Ð(r0   )	rÉ   rZ   rf   rÅ   rÈ   rÄ   rÏ   rË   rÇ   )r2   r3   r4   r5   r6   rd   rÙ   r£   r8   r9   r¦   r§   s   @@r-   r¾   r¾   È   s)   ù‡ € ÙGõD÷>eð e÷")÷ ")ð ")r0   r¾   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInstructBlipMLPi  c                ó(  <€ \         SV `  4        Wn        \        VP                  ,          V n        \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        R # ©N)rc   rd   rZ   r   Ú
hidden_actÚactivation_fnr   rÊ   re   Úintermediate_sizeÚfc1Úfc2rq   s   &&€r-   rd   ÚInstructBlipMLP.__init__  sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Õ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆŽr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r(   rÜ   r@   ©rT   ry   )r+   r,   s   "€r-   r.   ÚInstructBlipMLP.__annotate__  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rë   )rï   rí   rð   ©rK   rÜ   s   &&r-   r£   ÚInstructBlipMLP.forward  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr0   )rí   rZ   rï   rð   ©	r2   r3   r4   r5   rd   r£   r8   r9   r¦   r§   s   @@r-   ré   ré     s   ù‡ € õK÷÷ ð r0   ré   c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )ÚInstructBlipEncoderLayeri#  c                ó    <€ V ^8„  d   QhRS[ /# rY   ©r"   )r+   r,   s   "€r-   r.   Ú%InstructBlipEncoderLayer.__annotate__$  s   ø€ ÷ Sñ SÑ1ñ Sr0   c                óJ  <€ \         SV `  4        VP                  V n        \	        V4      V n        \        P                  ! V P                  VP                  R 7      V n	        \        V4      V n        \        P                  ! V P                  VP                  R 7      V n        R# ©©ÚepsN)rc   rd   re   rf   r¾   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1ré   ÚmlpÚlayer_norm2rq   s   &&€r-   rd   Ú!InstructBlipEncoderLayer.__init__$  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨vÓ6ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÔÜ" 6Ó*ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÖr0   c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r(   rÜ   r¹   r@   )rT   ry   r   r   rU   )r+   r,   s   "€r-   r.   rþ   -  s9   ø€ ÷ ñ á—|‘|ðñ Ñ+Õ,ðñ 
×	Ñ	ñ	r0   c                óÀ   € TpV P                  V4      pV P                  ! RR V/VB w  rW,           pTpV P                  V4      pV P                  V4      pW,           pV# )rÜ   r1   )r  r  r  r  )rK   rÜ   r¹   ÚresidualrŸ   s   &&,  r-   r£   Ú InstructBlipEncoderLayer.forward,  su   € ð !ˆà×(Ñ(¨Ó7ˆØŸ>š>ñ 
Ø'ð
àñ
Ñˆð &Õ0ˆØ ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆà%Õ0ˆàÐr0   )rf   r  r  r  r  )
r2   r3   r4   r5   rd   r   r£   r8   r9   r¦   r§   s   @@r-   rû   rû   #  s*   ù‡ € ÷Só Sð ÷ó ÷ð r0   rû   c                   óŽ   a a€ ] tR tRt oRtRtRtRtRtRt	Rt
Rt. ROt]P                  ! 4       V 3R l4       tV3R ltRtVtV ;t# )	ÚInstructBlipPreTrainedModeliC  ÚblipTc                óx  <€ \         SV `  V4       V P                  P                  p\	        V\
        4      '       dI   \        P                  ! VP                  RVR7       \        P                  ! VP                  RVR7       R# \	        V\        \        34      '       d#   \        P                  ! VP                  4       R# \	        V\        4      '       db   \        P                  ! VP                   \"        P$                  ! VP                   P&                  R,          4      P)                  R4      4       R# R# )zInitialize the weightsrÞ   )ÚmeanÚstdNrƒ   ©r^   rƒ   )rc   Ú_init_weightsrZ   Úinitializer_rangeÚ
isinstancerW   ÚinitÚtrunc_normal_rp   rk   Ú$InstructBlipForConditionalGenerationÚInstructBlipModelÚzeros_Úquery_tokensÚInstructBlipQFormerEmbeddingsÚcopy_Úposition_idsrT   Úaranger„   r�   )rK   r©   Úfactorrr   s   && €r-   r  Ú)InstructBlipPreTrainedModel._init_weightsY  sÓ   ø€ ô 	‰Ñ˜fÔ%Ø—‘×.Ñ.ˆÜ�fÔ:×;Ò;Ü×Ò˜v×8Ñ8¸sÈÕOÜ×Ò˜v×5Ñ5¸CÀV×LÜ˜Ô!EÔGXÐ Y×ZÒZÜ�KŠK˜×+Ñ+Ö,Ü˜Ô =×>Ò>Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhñ ?r0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rY   rý   )r+   r,   s   "€r-   r.   Ú(InstructBlipPreTrainedModel.__annotate__C  s   ø‡ ‚ áÑò r0   r1   )ÚimageÚtext)r  r¾   rû   ÚInstructBlipQFormerLayerÚ%InstructBlipQFormerMultiHeadAttentionÚInstructBlipQFormerSelfOutput)r2   r3   r4   r5   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_no_split_modulesrT   Úno_gradr  r7   r8   r9   r¦   r§   s   @@r-   r  r  C  sa   ù‡ € ð ÐØ(ÐØ&*Ð#Ø"&ÐØÐØ€NØÐà!ÐòÐð ‡]‚]ƒ_ô
ió ð
i÷/ … r0   r  c                   ó^   a a€ ] tR tRt oRtV3R lV 3R llt]V3R lR l4       tRtVt	V ;t
# )ÚInstructBlipEncoderih  zû
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`InstructBlipEncoderLayer`].

Args:
    config (`InstructBlipConfig`):
        The corresponding vision configuration for the `InstructBlipEncoder`.
c                ó    <€ V ^8„  d   QhRS[ /# rY   rý   )r+   r,   s   "€r-   r.   Ú InstructBlipEncoder.__annotate__r  s   ø€ ÷ ,ñ ,Ñ1ñ ,r0   c                óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        R V n	        R# u upi ©FN)
rc   rd   rZ   r   Ú
ModuleListÚrangeÚnum_hidden_layersrû   ÚlayersÚgradient_checkpointing)rK   rZ   rŸ   rr   s   && €r-   rd   ÚInstructBlipEncoder.__init__r  sT   ø€ Ü‰ÑÔØŒÜ—m’mÌuÐU[×UmÑUmÔOnÓ$oÑOnÈ!Ô%=¸fÖ%EÑOnÑ$oÓpˆŒØ&+ˆÖ#ùò %pó   ½A&c                óF   <€ V ^8„  d   QhRS[ S[,          RS[S[,          /# )r(   r¹   r@   )r   r   rA   r   )r+   r,   s   "€r-   r.   r8  y  s0   ø€ ÷ @ñ @ñ Ñ+Õ,ð@ñ 
‘Õ	 ñ	@r0   c                óV   € TpV P                    F  pV! V3/ VB pK  	  \        VR 7      # )©Úlast_hidden_state)r>  r   )rK   Úinputs_embedsr¹   rÜ   Úencoder_layers   &&,  r-   r£   ÚInstructBlipEncoder.forwardx  s9   € ð &ˆØ!Ÿ[œ[ˆMÙ)ØñàñŠMñ )ô °Ô?Ð?r0   )rZ   r?  r>  )r2   r3   r4   r5   r6   rd   r   r£   r8   r9   r¦   r§   s   @@r-   r6  r6  h  s0   ù‡ € ñ÷,ó ,ð ÷@ó ÷@ð @r0   r6  c                   ó¦   a a€ ] tR tRt oRtRtR]R]/tV3R lV 3R llt	]
]! RR7      ]RV3R	 lR
 ll4       4       4       tR tV3R ltRtVtV ;t# )ÚInstructBlipVisionModeliˆ  r•   rÜ   Ú
attentionsc                ó    <€ V ^8„  d   QhRS[ /# rY   r[   )r+   r,   s   "€r-   r.   Ú$InstructBlipVisionModel.__annotate__‘  s   ø€ ÷ 	ñ 	Ñ7ñ 	r0   c                óö   <€ \         SV `  V4       Wn        VP                  p\	        V4      V n        \        V4      V n        \        P                  ! W!P                  R 7      V n        V P                  4        R# r   )rc   rd   rZ   re   rW   ru   r6  Úencoderr   r  r  Úpost_layernormÚ	post_init)rK   rZ   rf   rr   s   && €r-   rd   Ú InstructBlipVisionModel.__init__‘  sY   ø€ Ü‰Ñ˜Ô ØŒØ×&Ñ&ˆ	ä6°vÓ>ˆŒÜ*¨6Ó2ˆŒÜ Ÿlšl¨9×:OÑ:OÔPˆÔà�‰Ör0   F)Útie_last_hidden_statesc          	      ót   <€ V ^8„  d   QhRS[ P                  R,          RS[RS[S[,          RS[S[,          /# )r(   r•   Nr’   r¹   r@   )rT   rU   r–   r   r   rA   r   )r+   r,   s   "€r-   r.   rM  Ÿ  sJ   ø€ ÷ 
ñ 
á×'Ñ'¨$Õ.ð
ñ #'ð
ñ Ñ+Õ,ð	
ñ
 
Ñ+Õ	+ñ
r0   c                óô   € Vf   \        R4      hV P                  WR7      pV P                  ! RRV/VB pVP                  pV P	                  V4      pVR,          pV P	                  V4      p\        VVR7      # )Nz You have to specify pixel_values)r’   rF  ©rE  Úpooler_outputr1   ©r‚   é    r‚   )rÆ   ru   rO  rE  rP  r   )rK   r•   r’   r¹   rÜ   Úencoder_outputsrE  Úpooled_outputs   &&&,    r-   r£   ÚInstructBlipVisionModel.forwardœ  s‘   € ð ÒÜÐ?Ó@Ð@àŸ™¨˜Óhˆà+/¯<ª<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ ×/Ñ/Ð0AÓBÐà)¨'Õ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
r0   c                ó   € V P                   # rë   )ru   rO   s   &r-   Úget_input_embeddingsÚ,InstructBlipVisionModel.get_input_embeddingsº  s   € Ø�‰Ðr0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rY   r[   )r+   r,   s   "€r-   r.   rM  ˆ  s   ø‡ ‚ ñ %Ñ$ò r0   )rZ   ru   rO  rP  )r&  ©NF)r2   r3   r4   r5   Úmain_input_namer,  rû   r¾   Ú_can_record_outputsrd   r   r   r   r£   r^  r7   r8   r9   r¦   r§   s   @@r-   rJ  rJ  ˆ  sk   ù‡ € Ø$€OØ!Ðð 	Ð1ØÐ+ðÐ÷
	ó 	ð  Ù¨EÔ2Ø÷
ñ 
ó ó 3ó  ð
ò6÷e … r0   rJ  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )r)  i¾  c                óø  <€ \         SV `  4        Wn        VP                  VP                  ,          ^ 8w  d;   \        VR4      '       g)   \        RVP                  VP                  3,          4      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        V P                  R,          V n
        RV n        VP                  V n        \        P                  ! VP                  V P                  4      V n        V'       dc   \        P                  ! VP"                  V P                  4      V n        \        P                  ! VP"                  V P                  4      V n        R# \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        R# )rY  Úembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)FNrÂ   )rc   rd   rZ   re   rÃ   ÚhasattrrÆ   rz   Úattention_head_sizeÚall_head_sizer®   rÈ   Úattention_probs_dropout_probrÉ   r   rÊ   rª   Úencoder_hidden_sizer«   r¬   ©rK   rZ   Úis_cross_attentionrr   s   &&&€r-   rd   Ú.InstructBlipQFormerMultiHeadAttention.__init__¿  sk  ø€ Ü‰ÑÔØŒØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ^Ø×%Ñ% v×'AÑ'AÐBõCóð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔØ×/Ñ/°Õ5ˆŒØˆŒØ!'×!DÑ!DˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ßÜ—y’y ×!;Ñ!;¸T×=OÑ=OÓPˆDŒHÜŸš 6×#=Ñ#=¸t×?QÑ?QÓRˆDŽJä—y’y ×!3Ñ!3°T×5GÑ5GÓHˆDŒHÜŸš 6×#5Ñ#5°t×7IÑ7IÓJˆDŽJr0   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©r(   r¹   ©r   r   )r+   r,   s   "€r-   r.   Ú2InstructBlipQFormerMultiHeadAttention.__annotate__×  s   ø€ ÷ ,)ñ ,)ñ Ñ+Õ,ñ,)r0   c                ó  € VR JpVP                   R R p. VORNV P                  N5pV'       d   Tp	TpMTp	. V	P                   R R ORNV P                  N5p
V P                  V4      P                  V4      P	                  ^^4      pV P                  V	4      P                  V
4      P	                  ^^4      pV P                  V	4      P                  V
4      P	                  ^^4      p\        P                  ! V P                  P                  \        4      pT! V VVVV3RV P                  '       g   RMV P                  RV P                  /VB w  ppVP                  ! . VP                   R,          ORN5!  P!                  4       pVV3# )Nr¯   rÞ   r®   :Nr(   Nrƒ   )r„   rh  rª   r‹   rœ   r«   r¬   r   rß   rZ   rà   r¼   r´   rÉ   r®   r‡   r¸   )rK   rÜ   r­   Úencoder_hidden_statesÚencoder_attention_maskr¹   rm  Úinput_shapeÚhidden_shapeÚcurrent_statesÚkv_shapeÚquery_layerÚ	key_layerÚvalue_layerræ   r»   rº   s   &&&&&,           r-   r£   Ú-InstructBlipQFormerMultiHeadAttention.forward×  s“  € ð 3¸$Ð>Ðà#×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆçØ2ˆNØ3‰Nà*ˆNàM�^×)Ñ)¨#¨2Ð.ÐM°ÐM°D×4LÑ4LÑMˆØ—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆØ—H‘H˜^Ó,×1Ñ1°(Ó;×EÑEÀaÈÓKˆ	Ø—j‘j Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)ÐE¨;×+<Ñ+<¸RÕ+@ÐEÀ"ÓE×PÑPÓRˆØ˜LÐ(Ð(r0   )
ri  rÉ   rh  rZ   rÈ   r«   rÃ   rª   r®   r¬   r¥   ©NNNrù   r§   s   @@r-   r)  r)  ¾  s   ù‡ € ÷K÷0,)÷ ,)ò ,)r0   r)  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )r*  i  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# r   )rc   rd   r   rÊ   re   Údenser  r  ÚDropoutÚhidden_dropout_probr¯   rq   s   &&€r-   rd   Ú&InstructBlipQFormerSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr0   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r(   rÜ   Úinput_tensorr@   rô   )r+   r,   s   "€r-   r.   Ú*InstructBlipQFormerSelfOutput.__annotate__  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ r0   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rë   ©r�  r¯   r  ©rK   rÜ   r‡  s   &&&r-   r£   Ú%InstructBlipQFormerSelfOutput.forward  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr0   ©r  r�  r¯   rù   r§   s   @@r-   r*  r*    ó   ù‡ € õ>÷÷ ð r0   r*  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚInstructBlipQFormerAttentioni  c                ód   <€ \         SV `  4        \        W4      V n        \	        V4      V n        R # rë   )rc   rd   r)  Ú	attentionr*  Úoutputrl  s   &&&€r-   rd   Ú%InstructBlipQFormerAttention.__init__  s&   ø€ Ü‰ÑÔÜ>¸vÓZˆŒÜ3°FÓ;ˆŽr0   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[ P                  /# )r(   rÜ   r­   Nrt  ru  r¹   r@   )rT   ry   rU   r   r   )r+   r,   s   "€r-   r.   Ú)InstructBlipQFormerAttention.__annotate__  sv   ø€ ÷  ñ  á—|‘|ð ñ ×)Ñ)¨DÕ0ð ñ  %×0Ñ0°4Õ7ð	 ñ
 !&× 1Ñ 1°DÕ 8ð ñ Ñ+Õ,ð ñ 
�‰ñ r0   c                ó`   € V P                   ! RR VRVRVRV/VB w  rgV P                  Wa4      pV# )rÜ   r­   rt  ru  r1   ©r”  r•  )	rK   rÜ   r­   rt  ru  r¹   r»   rŸ   Úattention_outputs	   &&&&&,   r-   r£   Ú$InstructBlipQFormerAttention.forward  sY   € ð Ÿšñ 
Ø'ð
à)ð
ð #8ð
ð $:ð	
ð
 ñ
‰ˆð  Ÿ;™; {ÓBÐØÐr0   rš  r¥   r~  rù   r§   s   @@r-   r’  r’    s   ù‡ € ÷<÷
 ÷  ò  r0   r’  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInstructBlipQFormerIntermediatei0  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rë   )rc   rd   r   rÊ   re   rî   r�  r  rì   Ústrr   Úintermediate_act_fnrq   s   &&€r-   rd   Ú(InstructBlipQFormerIntermediate.__init__1  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ró   rô   )r+   r,   s   "€r-   r.   Ú,InstructBlipQFormerIntermediate.__annotate__9  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r0   c                óJ   € V P                  V4      pV P                  V4      pV# rë   ©r�  r¡  r÷   s   &&r-   r£   Ú'InstructBlipQFormerIntermediate.forward9  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr0   r¦  rù   r§   s   @@r-   rž  rž  0  s   ù‡ € õ9÷÷ ð r0   rž  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInstructBlipQFormerOutputi@  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# r   )rc   rd   r   rÊ   rî   re   r�  r  r  r‚  rƒ  r¯   rq   s   &&€r-   rd   Ú"InstructBlipQFormerOutput.__init__A  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr0   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# r†  rô   )r+   r,   s   "€r-   r.   Ú&InstructBlipQFormerOutput.__annotate__G  r‰  r0   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rë   r‹  rŒ  s   &&&r-   r£   Ú!InstructBlipQFormerOutput.forwardG  rŽ  r0   r�  rù   r§   s   @@r-   r©  r©  @  r�  r0   r©  c                   óT   a a€ ] tR tRt oV 3R ltRV3R lR lltR tR tRtVt	V ;t
# )	r(  iN  c                óp  <€ \         SV `  4        VP                  V n        ^V n        \	        V4      V n        W n        W!P                  ,          ^ 8X  d   \	        VRR7      V n        RV n	        MRV n	        \        V4      V n        \        V4      V n        \        V4      V n        \        V4      V n        R# )r^   T)rm  FN)rc   rd   Úchunk_size_feed_forwardÚseq_len_dimr’  r”  Ú	layer_idxÚcross_attention_frequencyÚcrossattentionÚhas_cross_attentionrž  Úintermediater©  r•  Úintermediate_queryÚoutput_query©rK   rZ   r´  rr   s   &&&€r-   rd   Ú!InstructBlipQFormerLayer.__init__O  s™   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ5°fÓ=ˆŒà"Œà×7Ñ7Õ7¸1Ô<Ü">¸vÐZ^Ô"_ˆDÔØ'+ˆDÕ$à',ˆDÔ$ä;¸FÓCˆÔÜ/°Ó7ˆŒä"AÀ&Ó"IˆÔÜ5°fÓ=ˆÖr0   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rp  rq  )r+   r,   s   "€r-   r.   Ú%InstructBlipQFormerLayer.__annotate__c  s   ø€ ÷ 3ñ 3ñ Ñ+Õ,ñ3r0   c           	     ó†  € V P                   ! V3R V/VB pV^ 8”  dù   VRRV1R3,          pV P                  '       d)   Vf   \        R4      hV P                  ! V3R VRVRV/VB p\	        V P
                  V P                  V P                  V4      p	VP                  ^,          V8”  dk   \	        V P                  V P                  V P                  VRVR1R3,          4      P                  V	P                  4      p
\        P                  ! Wš.^R7      p	V	# \	        V P                  V P                  V P                  V4      p	V	# )r­   r‚   Nz>encoder_hidden_states must be given for cross-attention layersrt  ru  r€   )r”  r·  rÆ   r¶  r   Úfeed_forward_chunk_queryr²  r³  r„   Úfeed_forward_chunkrš   ÚdevicerT   rŒ   )rK   rÜ   r­   rt  ru  Úquery_lengthr¹   r›  Úquery_attention_outputÚlayer_outputÚlayer_output_texts   &&&&&&,    r-   r£   Ú InstructBlipQFormerLayer.forwardc  s|  € ð  Ÿ>š>Øñ
à)ð
ð ñ
Ðð ˜!ÔØ%5°a¸¸,¸ÈÐ6IÕ%JÐ"à×'×'Ð'Ø(Ò0Ü$Ð%eÓfÐfØ)-×)<Ò)<Ø*ñ*à#1ð*ð +@ð*ð ,Bð	*ð
 ñ*Ð&ô 5Ø×-Ñ-Ø×,Ñ,Ø× Ñ Ø&ó	ˆLð  ×%Ñ% aÕ(¨<Ô7Ü$=Ø×+Ñ+Ø×0Ñ0Ø×$Ñ$Ø$ Q¨©°qÐ%8Õ9ó	%÷
 ‘"�\×(Ñ(Ó)ð "ô  %Ÿyšy¨,Ð)JÐPQÔR�ð Ðô 5Ø×'Ñ'Ø×,Ñ,Ø× Ñ Ø ó	ˆLð Ðr0   c                óJ   € V P                  V4      pV P                  W!4      pV# rë   )r¸  r•  ©rK   r›  Úintermediate_outputrÅ  s   &&  r-   rÁ  Ú+InstructBlipQFormerLayer.feed_forward_chunk˜  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐr0   c                óJ   € V P                  V4      pV P                  W!4      pV# rë   )r¹  rº  rÉ  s   &&  r-   rÀ  Ú1InstructBlipQFormerLayer.feed_forward_chunk_query�  s+   € Ø"×5Ñ5Ð6FÓGÐØ×(Ñ(Ð)<ÓOˆØÐr0   )
r”  r²  r¶  r·  r¸  r¹  r´  r•  rº  r³  ©NNNrY  )r2   r3   r4   r5   rd   r£   rÁ  rÀ  r8   r9   r¦   r§   s   @@r-   r(  r(  N  s$   ù‡ € õ>÷(3ò 3òj÷
ò r0   r(  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚInstructBlipQFormerEncoderi¤  c           	     óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        R V n	        R# u upi r:  )
rc   rd   rZ   r   r;  r<  r=  r(  Úlayerr?  r»  s   && €r-   rd   Ú#InstructBlipQFormerEncoder.__init__¥  sY   ø€ Ü‰ÑÔØŒÜ—]’]ÜJOÐPV×PhÑPhÔJiÓjÑJi¸YÔ% fÖ8ÑJiÑjó
ˆŒ
ð ',ˆÖ#ùò krA  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rp  rq  )r+   r,   s   "€r-   r.   Ú'InstructBlipQFormerEncoder.__annotate__®  s   ø€ ÷ 
ñ 
ñ Ñ+Õ,ñ
r0   c                óª   € \        V P                  P                  4       F%  pV P                  V,          pV! VVV3R VRV/VB pK'  	  \	        VR7      # )ru  rÃ  rD  )r<  rZ   r=  rÒ  r   )	rK   rÜ   r­   rt  ru  rÃ  r¹   ÚiÚlayer_modules	   &&&&&&,  r-   r£   Ú"InstructBlipQFormerEncoder.forward­  sm   € ô �t—{‘{×4Ñ4Ö5ˆAØŸ:™: a�=ˆLá(ØØØ%ñð (>ð	ð
 *ðð ñŠMñ 6ô 9Ø+ô
ð 	
r0   )rZ   r?  rÒ  rÎ  )
r2   r3   r4   r5   rd   r   r£   r8   r9   r¦   r§   s   @@r-   rÐ  rÐ  ¤  s(   ù‡ € õ,ð ÷
ñ 
ó ÷
ð 
r0   rÐ  c                   ó@   a a€ ] tR tRt oRtV 3R ltRR ltRtVtV ;t	# )r  iÈ  z;Construct the embeddings from word and position embeddings.c                óH  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  VP
                  4      V n	        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P                   ! \"        P$                  ! VP                  4      P'                  R4      RR7      V n        Wn        R# ))Úpadding_idxr  F)Ú
persistentNr  )rc   rd   r   Ú	EmbeddingÚ
vocab_sizere   Úpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsr  r  Ú	layernormr‚  rƒ  r¯   ÚBufferrT   r!  r�   r   rZ   rq   s   &&€r-   rd   Ú&InstructBlipQFormerEmbeddings.__init__Ë  sÀ   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÔàŽr0   c                ó  € Ve   VP                  4       ^,          pM^ pVf,   V P                  RWEV,           13,          P                  4       pVeb   V P                  V4      pV P	                  VP                  VP                  4      4      pWg,           pVe   \        P                  ! W63^R7      pMTpVP                  V P                  P                  P                  4      pV P                  V4      pV P                  V4      pV# )Nr‚   r€   )r}   r   Úclonerá  rã  rš   rÂ  rT   rŒ   rä  r™   r˜   r¯   )rK   Ú	input_idsr   Úquery_embedsÚpast_key_values_lengthÚ
seq_lengthru   rã  s   &&&&&   r-   r£   Ú%InstructBlipQFormerEmbeddings.forwardØ  së   € ð Ò Ø"Ÿ™Ó)¨!Õ,‰JàˆJàÒØ×,Ñ,¨QÐ0FÐVlÕIlÐ0lÐ-lÕm×sÑsÓuˆLàÒ Ø×-Ñ-¨iÓ8ˆJà"&×":Ñ":¸<¿?¹?È:×K\ÑK\Ó;]Ó"^ÐØ#Õ9ˆJàÒ'Ü"ŸYšY¨Ð'AÀqÔI�
øà%ˆJà—]‘] 4§>¡>×#8Ñ#8×#>Ñ#>Ó?ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr0   )rZ   r¯   rä  rã  r   rá  rÎ  )
r2   r3   r4   r5   r6   rd   r£   r8   r9   r¦   r§   s   @@r-   r  r  È  s   ù‡ € ÙEõ÷ô r0   r  c                   óÊ   a a€ ] tR tRt oRtRtRtRtRtR]	R]
! ]^RR7      .R]
! ]^R	R7      ./tV3R
 lV 3R lltR tR t]]]RV3R lR ll4       4       4       tRtVtV ;t# )ÚInstructBlipQFormerModeliø  z�
Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.
TrÜ   rK  z
.attention)ÚindexÚ
layer_nameÚcross_attentionsz.crossattentionc                ó    <€ V ^8„  d   QhRS[ /# rY   )r#   )r+   r,   s   "€r-   r.   Ú%InstructBlipQFormerModel.__annotate__  s   ø€ ÷ ñ Ñ8ñ r0   c                ó’   <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        V P                  4        R # rë   )rc   rd   rZ   r  ru   rÐ  rO  rQ  rq   s   &&€r-   rd   Ú!InstructBlipQFormerModel.__init__  s7   ø€ Ü‰Ñ˜Ô ØŒä7¸Ó?ˆŒä1°&Ó9ˆŒà�‰Ör0   c                ó.   € V P                   P                  # rë   ©ru   rá  rO   s   &r-   r^  Ú-InstructBlipQFormerModel.get_input_embeddings  s   € Ø�‰×.Ñ.Ð.r0   c                ó&   € WP                   n        R # rë   rø  )rK   r¬   s   &&r-   Úset_input_embeddingsÚ-InstructBlipQFormerModel.set_input_embeddings  s   € Ø*/�‰Ö'r0   c                óL  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          R	S[S[ P                  ,          S[,          /# )
r(   ré  r­   Nr   rê  rt  ru  r¹   r@   )rT   Ú
LongTensorrU   ry   r   r   rA   r   )r+   r,   s   "€r-   r.   rô     s­   ø€ ÷ 9
ñ 9
á×#Ñ#ð9
ñ ×)Ñ)¨DÕ0ð9
ñ ×&Ñ&¨Õ-ð	9
ñ
 —l‘l TÕ)ð9
ñ  %×0Ñ0°4Õ7ð9
ñ !&× 1Ñ 1°DÕ 8ð9
ñ Ñ+Õ,ð9
ñ 
‰u× Ñ Õ	!Ñ$PÕ	Pñ9
r0   c                ó”  € Vf   Vf   \        R4      hVe   VP                  ^,          M^ pV P                  VVVR7      p	\        V P                  V	VR7      pVe1   V^ 8”  d   V	RRV1R3,          MT	p
\        V P                  V
VVR7      pV P
                  ! V	3RVRVR	VR
V/VB pVP                  pVR,          p\        VVR7      # )a  
query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
    Hidden states to be used in the attention computation. If cross-attention,
    will be used for the query (i.e., key and value will use the encoder_hidden_states).
Nz7You have to specify query_embeds when input_ids is None)ré  r   rê  )rZ   rF  r­   r‚   )rZ   rF  r­   rt  r­   rt  ru  rÃ  rV  rX  )rÆ   r„   ru   r	   rZ   rO  rE  r   )rK   ré  r­   r   rê  rt  ru  r¹   rÃ  Úembedding_outputÚquery_embedding_outputrZ  Úsequence_outputr[  s   &&&&&&&,      r-   r£   Ú InstructBlipQFormerModel.forward  s  € ð$ Ò Ò!5ÜÐVÓWÐWà0<Ò0H�|×)Ñ)¨!Ö,ÈaˆàŸ?™?ØØ%Ø%ð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð "Ò-ð O[Ð]^ÔN^Ð%5°a¸¸,¸ÈÐ6IÖ%JÐdtÐ"Ü%>Ø—{‘{Ø4Ø5Ø&;ô	&Ð"ð ,0¯<ª<Øñ,
à)ð,
ð #8ð,
ð $:ð	,
ð
 &ð,
ð ñ,
ˆð *×;Ñ;ˆØ'¨Õ0ˆä;Ø-Ø'ô
ð 	
r0   )rZ   ru   rO  )NNNNN)r2   r3   r4   r5   r6   r.  r0  r/  r1  r(  r   r)  rc  rd   r^  rû  r   r   r   r£   r8   r9   r¦   r§   s   @@r-   rï  rï  ø  s£   ù‡ € ñð
 #'ÐØ€NØÐØÐð 	Ð1ØÙÐ@ÈÐVbÔcð
ð 	ÙÐ@ÈÐVgÔhð
ðÐ÷ó ò/ò0ð  ØØ÷9
ñ 9
ó ó ó  ÷9
ð 9
r0   rï  z[
    InstructBLIP base Model consisting of language model, qformer and vision encoder.
    c                   óŠ   a a€ ] tR tRt oRtR.tV3R lV 3R lltR tV3R lR lt]	]
RV3R	 lR
 ll4       4       tRtVtV ;t# )r  i\  r•   r  c                ó    <€ V ^8„  d   QhRS[ /# rY   rý   )r+   r,   s   "€r-   r.   ÚInstructBlipModel.__annotate__e  s   ø€ ÷ ñ Ñ1ñ r0   c                ó  <€ \         SV `  V4       \        VP                  4      V n        \
        P                  ! \        P                  ! ^VP                  VP                  P                  4      4      V n        \        VP                  4      V n        \
        P                  ! VP                  P                  VP                   P                  4      V n        \$        P&                  ! VP                   4      V n        V P+                  4        R# ©r^   N)rc   rd   rJ  Úvision_configÚvision_modelr   ri   rT   rÍ   Únum_query_tokensÚqformer_configre   r  rï  ÚqformerrÊ   Útext_configÚlanguage_projectionr   Úfrom_configÚlanguage_modelrQ  rq   s   &&€r-   rd   ÚInstructBlipModel.__init__e  s·   ø€ Ü‰Ñ˜Ô ä3°F×4HÑ4HÓIˆÔÜŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ/°×0EÑ0EÓFˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ Ü'×3Ò3°F×4FÑ4FÓGˆÔð 	�‰Ör0   c                ó.  € V P                   p\        V4      ^8”  d@   RV9  d9   \        P                  P	                  4       ^8”  d   \
        P                  R4       \        V P                  R4      '       d   RV P                  P                  n
        R# R# ©z•
Some pre-processing hacks to make the model `accelerate` compatible. Check
https://github.com/huggingface/transformers/pull/21707 for more details.
r  a   The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models.Ú_hf_hookTN©Úhf_device_mapÚlenrT   ÚcudaÚdevice_countÚloggerÚwarningrg  r  r  Úio_same_device©rK   r  s   & r-   Ú_preprocess_accelerateÚ(InstructBlipModel._preprocess_accelerater  ó|   € ð
 ×*Ñ*ˆäˆ}Ó Ô!Ð&6¸mÔ&KÔPU×PZÑPZ×PgÑPgÓPiÐlmÔPmä�N‰NðMôô �4×&Ñ&¨
×3Ò3Ø:>ˆD×Ñ×(Ñ(Ö7ñ 4r0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r(   ré  rF  ©rT   rþ  rU   )r+   r,   s   "€r-   r.   r  †  ó'   ø€ ÷ "ñ "©e×.>Ñ.>ð "Éu×O`ÑO`ñ "r0   c           
     ón  € Vfo   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pMWP                  P                  8H  pVP                  R4      P                  VP                  4      pV# ©zJ
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
©r˜   rÂ  r1   rƒ   ©
r^  rT   ÚfullrZ   Úimage_token_idÚlongrÂ  ÚallÚ	unsqueezerš   ©rK   ré  rF  Úspecial_image_masks   &&& r-   Úget_placeholder_maskÚ&InstructBlipModel.get_placeholder_mask†  ó˜   € ð ÒØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÔió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¯k©k×.HÑ.HÑ!HÐà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ!Ð!r0   c                óp  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[RS[S[,          RS[S[,          /# )r(   r•   Úqformer_input_idsÚqformer_attention_maskNré  r­   Údecoder_input_idsÚdecoder_attention_maskrF  r’   r¹   r@   )	rT   rU   rþ  ry   r–   r   r
   rA   r=   )r+   r,   s   "€r-   r.   r  —  sÞ   ø€ ÷ _
ñ _
á×'Ñ'ð_
ñ !×,Ñ,ð_
ñ !&× 0Ñ 0°4Õ 7ð	_
ñ
 ×$Ñ$ tÕ+ð_
ñ ×(Ñ(¨4Õ/ð_
ñ !×+Ñ+¨dÕ2ð_
ñ !&× 0Ñ 0°4Õ 7ð_
ñ —|‘| dÕ*ð_
ñ #'ð_
ñ Ñ-Õ.ð_
ñ 
Ñ@Õ	@ñ_
r0   c
                ó¨  € V P                   ! RRVRV	/V
B pV^ ,          p\        P                  ! VP                  4       RR \        P                  VP
                  R7      pV P                  P                  VP                  ^ ,          RR4      p\        P                  ! VP                  4       RR \        P                  VP
                  R7      pVf   \        P                  ! V4      pVP                  VP
                  4      p\        P                  ! Wó.^R7      pV P                  ! RRVRVRVR	VR
V/V
B pV^ ,          RRVP                  ^4      1R3,          pVf;   V P                  P                  4       ! V4      pVf   \        P                  ! V4      pV P                  V4      pVP                  VP
                  VP                   4      pV P#                  WHR7      pVP%                  VV4      pV P&                  P(                  '       d   V P                  ! RRVRV/V
B pMV P                  ! RRVRVRVRV/V
B p\+        VVVR7      # )aK  
qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
    to serve as text prompt, which the Q-Former model will encode.

    Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
    details.

    [What are input IDs?](../glossary#input-ids)
qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    Only relevant in case an encoder-decoder language model (like T5) is used.
r•   r’   Nr(  r€   ré  r­   rê  rt  ru  r‚   ©rF  rF  r7  r8  rE   r1   rƒ   )r
  rT   Úonesr}   r,  rÂ  r  r�   r„   Ú	ones_likerš   rŒ   r  r  r^  r  r˜   r1  Úmasked_scatterrZ   Úuse_decoder_only_language_modelr=   )rK   r•   r5  r6  ré  r­   r7  r8  rF  r’   r¹   r)   Úimage_embedsÚimage_attention_maskr  Úquery_attention_maskÚquery_outputsÚquery_outputÚlanguage_model_inputsr0  Úoutputss   &&&&&&&&&&,          r-   r£   ÚInstructBlipModel.forward•  s’  € ðP ×*Ò*ñ 
Ø%ð
à%=ð
ð ñ
ˆð
 & aÕ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐØ!Ò)Ü%*§_¢_Ð5FÓ%GÐ"Ø!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@Ð+YÐ_`Ô!aÐØŸšñ 
Ø'ð
à1ð
ð &ð
ð #/ð	
ð
 $8ð
ð ñ
ˆð % QÕ'¨Ð+A¨\×->Ñ->¸qÓ-AÐ+AÀ1Ð(DÕEˆàÒ Ø ×/Ñ/×DÑDÔFÀyÓQˆMØÒ%Ü!&§¢°Ó!;�ð !%× 8Ñ 8¸Ó FÐØ 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ó^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ð6Ø×)Ò)ñ Ø+ðà-ðð ñ‰Gð ×)Ò)ñ Ø+ðà-ðð #4ðð (>ð	ð
 ñˆGô ?Ø)Ø)Ø#*ô
ð 	
r0   ©r  r  r  r  r
  )NNNNNNF)r2   r3   r4   r5   rb  Ú_keep_in_fp32_modulesrd   r  r1  r   r   r£   r8   r9   r¦   r§   s   @@r-   r  r  \  sX   ù‡ € ð %€OØ+Ð,Ð÷ó ò?÷("ð "ð Ø÷_
ñ _
ó ó ÷_
ð _
r0   r  aª  
    InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                   ó0  a a€ ] tR tRt oRtRtR.tV3R lV 3R lltR tV3R lR	 lt	RV 3R
 llt
R tR t]]RV3R lR ll4       4       tV3R lR lt]]RV3R lR ll4       4       t]P&                  ! 4       RV3R lR ll4       tV3R ltRtVtV ;t# )r  iù  r•   Tr  c                ó    <€ V ^8„  d   QhRS[ /# rY   rý   )r+   r,   s   "€r-   r.   Ú1InstructBlipForConditionalGeneration.__annotate__	  s   ø€ ÷ ñ Ñ1ñ r0   c                ó®  <€ \         SV `  V4       \        P                  VP                  4      V n        \        P                  ! \        P                  ! ^VP                  VP                  P                  4      4      V n        \        P                  VP                  4      V n        \        P                   ! VP                  P                  VP"                  P                  4      V n        VP&                  '       d"   \(        P*                  ! VP"                  4      pM \,        P*                  ! VP"                  4      pW n        V P1                  4        R# r  )rc   rd   rJ  Ú_from_configr	  r
  r   ri   rT   rÍ   r  r  re   r  rï  r  rÊ   r  r  r>  r    r  r!   r  rQ  )rK   rZ   r  rr   s   && €r-   rd   Ú-InstructBlipForConditionalGeneration.__init__	  sè   ø€ Ü‰Ñ˜Ô ä3×@Ñ@À×AUÑAUÓVˆÔäŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ/×<Ñ<¸V×=RÑ=RÓSˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ à×1×1Ð1Ü1×=Ò=¸f×>PÑ>PÓQ‰Nä2×>Ò>¸v×?QÑ?QÓRˆNà,Ôð 	�‰Ör0   c                ó<   € V P                   P                  V4       R # rë   )r  Úset_output_embeddings)rK   Únew_embeddingss   &&r-   rP  Ú:InstructBlipForConditionalGeneration.set_output_embeddings  s   € Ø×Ñ×1Ñ1°.ÖAr0   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r?   )r   r°   )r+   r,   s   "€r-   r.   rK     s   ø€ ÷ ;ñ ;¡r§y¡yñ ;r0   c                ó6   € V P                   P                  4       # rë   )r  Úget_output_embeddingsrO   s   &r-   rU  Ú:InstructBlipForConditionalGeneration.get_output_embeddings   s   € Ø×"Ñ"×8Ñ8Ó:Ð:r0   c                ó`   <€ Vf   V P                   P                  4       # \        SV `  VR7      # )N)Úmodality)r  Úget_encoderrc   )rK   rX  rr   s   &&€r-   rY  Ú0InstructBlipForConditionalGeneration.get_encoder#  s1   ø€ ØÒØ×&Ñ&×2Ñ2Ó4Ð4ä‘7Ñ&°Ð&Ó9Ð9r0   c                ó6   € V P                   P                  4       # rë   )r  Úget_decoderrO   s   &r-   r\  Ú0InstructBlipForConditionalGeneration.get_decoder)  s   € Ø×"Ñ"×.Ñ.Ó0Ð0r0   c                ó.  € V P                   p\        V4      ^8”  d@   RV9  d9   \        P                  P	                  4       ^8”  d   \
        P                  R4       \        V P                  R4      '       d   RV P                  P                  n
        R# R# r  r  r  s   & r-   r  Ú;InstructBlipForConditionalGeneration._preprocess_accelerate-  r!  r0   c                ó¶   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[,          RS[S[,          /# )r(   r•   r5  r6  Nr’   r¹   r@   )rT   rU   rþ  r–   r   r   rA   r&   )r+   r,   s   "€r-   r.   rK  C  sv   ø€ ÷ @ñ @á×'Ñ'ð@ñ !×+Ñ+ð@ñ !&× 0Ñ 0°4Õ 7ð	@ñ
 #'¨¥+ð@ñ Ñ+Õ,ð@ñ 
Ñ8Õ	8ñ@r0   c                ó$  € V P                   ! RRVRVRR/VB p\        R/ VBRV/B pV^ ,          p\        P                  ! VP	                  4       RR \        P
                  VP                  R7      pV P                  P                  VP                  ^ ,          RR4      p	\        P                  ! V	P	                  4       RR \        P
                  VP                  R7      p
Vf   \        P                  ! V4      pVP                  V
P                  4      p\        P                  ! W£.^R7      pV P                  ! RR	VR
VRV	RVRVRR/VB pW¶n        V^ ,          RRV	P	                  ^4      1R3,          pV P                  V4      pWÖn        V# )a¡  
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
    The tensors corresponding to the input images.
qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
    to serve as text prompt, which the Q-Former model will encode.

    Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
    details.

    [What are input IDs?](../glossary#input-ids)
qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
r•   r’   Úreturn_dictTr)   Nr(  r€   ré  r­   rê  rt  ru  r‚   r1   rƒ   )r
  r&   rT   r;  r}   r,  rÂ  r  r�   r„   r<  rš   rŒ   r  r*   r  rW  )rK   r•   r5  r6  r’   r¹   r)   r?  r@  r  rA  r*   rC  Úimage_featuress   &&&&&,        r-   Úget_image_featuresÚ7InstructBlipForConditionalGeneration.get_image_featuresA  s¾  € ð> 6:×5FÒ5Fñ 6
Ø%ð6
à%=ð6
ð ð6
ð ñ	6
ˆô AÑqÀ>ÑqÐbpÒqˆØ% aÕ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐØ!Ò)Ü%*§_¢_Ð5FÓ%GÐ"Ø!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@Ð+YÐ_`Ô!aÐØŸ,š,ñ 
Ø'ð
à1ð
ð &ð
ð #/ð	
ð
 $8ð
ð ð
ð ñ
ˆð *9Ô&Ø& qÕ)¨!Ð-C¨|×/@Ñ/@ÀÓ/CÐ-CÀQÐ*FÕGˆð ×1Ñ1°,Ó?ˆØ'5Ô$àÐr0   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r#  r$  )r+   r,   s   "€r-   r.   rK  …  r%  r0   c           
     ón  € Vfo   W P                  4       ! \        P                  ! RV P                  P                  \        P
                  VP                  R7      4      8H  pVP                  R4      pMWP                  P                  8H  pVP                  R4      P                  VP                  4      pV# r'  r)  r/  s   &&& r-   r1  Ú9InstructBlipForConditionalGeneration.get_placeholder_mask…  r3  r0   c                ó˜  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[RS[S[,          RS[S[,          /# )r(   r•   r5  r6  Nré  r­   r7  r8  rF  Úlabelsr’   r¹   r@   )rT   rU   rþ  r–   r   r   rA   r=   )r+   r,   s   "€r-   r.   rK  –  sö   ø€ ÷ B
ñ B
á×'Ñ'ðB
ñ !×,Ñ,ðB
ñ !&× 0Ñ 0°4Õ 7ð	B
ñ
 ×$Ñ$ tÕ+ðB
ñ ×(Ñ(¨4Õ/ðB
ñ !×+Ñ+¨dÕ2ðB
ñ !&× 0Ñ 0°4Õ 7ðB
ñ ×(Ñ(¨4Õ/ðB
ñ × Ñ  4Õ'ðB
ñ #'ðB
ñ Ñ+Õ,ðB
ñ 
Ñ@Õ	@ñB
r0   c                óô  € V P                  VVVV
RR7      pVP                  pVP                  pVP                  pVf   V P	                  4       ! V4      pVf   \
        P                  ! V4      pVP                  VP                  VP                  4      pV P                  WHR7      pVP                  VV4      pV P                  P                  '       d]   V P                  ! RRVRV/VB pV^ ,          pRpV	e7   V P                  ! RRVRV	R	V P                  P                   P"                  /VB pM9RVR
&   V P                  ! RRVRVRVRVRV	/VB pVP$                  pVP&                  p\)        VVVVVR7      # )a  
qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
    to serve as text prompt, which the Q-Former model will encode.

    Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
    details.

    [What are input IDs?](../glossary#input-ids)
qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    Only relevant in case an encoder-decoder language model (like T5) is used.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size -
    1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
    config.vocab_size]`

Examples:

```python
>>> from transformers import InstructBlipProcessor, InstructBlipForConditionalGeneration
>>> import torch
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO

>>> model = InstructBlipForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b")
>>> processor = InstructBlipProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

>>> device = "cuda" if torch.cuda.is_available() else "cpu"
>>> model.to(device)  # doctest: +IGNORE_RESULT

>>> url = "https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read())).convert("RGB")
>>> prompt = "What is unusual about this image?"
>>> inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)

>>> outputs = model.generate(
...     **inputs,
...     do_sample=False,
...     num_beams=5,
...     max_length=256,
...     min_length=1,
...     top_p=0.9,
...     repetition_penalty=1.5,
...     length_penalty=1.0,
...     temperature=1,
... )
>>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
>>> print(generated_text)
The unusual aspect of this image is that a man is ironing clothes on the back of a yellow SUV, which is parked in the middle of a busy city street. This is an unconventional approach to ironing clothes, as it requires the man to balance himself and his ironing equipment on top of the vehicle while navigating through traffic. Additionally, the presence of taxis and other vehicles in the scene further emphasizes the unusual nature of this situation.
```T©r5  r6  r’   rb  Nr:  rF  r­   rS   rj  rß  rb  r7  r8  )rR   rS   r)   r*   rF   r1   )rd  rW  r*   r)   r^  rT   r<  rš   rÂ  r˜   r1  r=  rZ   r>  r  Úloss_functionr  rß  rR   rS   r=   )rK   r•   r5  r6  ré  r­   r7  r8  rF  rj  r’   r¹   rc  rD  r*   r)   r0  rE  rS   rR   s   &&&&&&&&&&&,        r-   r£   Ú,InstructBlipForConditionalGeneration.forward”  sÛ  € ð^ CG×BYÑBYØØ/Ø#9Ø%=Øð CZó C
ˆð !/× <Ñ <ÐØ(×8Ñ8ˆØ'×6Ñ6ˆàÒ Ø ×5Ñ5Ô7¸	ÓBˆMàÒ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ó^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ð6Ø×)Ò)ñ Ø+ðà-ðð ñˆGð
 ˜Q•ZˆFØˆDØÒ!Ø×)Ò)ñ Ø!ðØ*0ðØ=A¿[¹[×=TÑ=T×=_Ñ=_ðØciñ�øð
 %)ˆF�=Ñ!Ø×)Ò)ñ Ø+ðà-ðð #4ðð (>ð	ð
 ðð ñˆGð —<‘<ˆDØ—^‘^ˆFä>ØØØ)Ø+Ø#*ô
ð 	
r0   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R	S[ P                  /# )
r(   r•   r5  Nr6  ré  r­   rF  r’   r@   )rT   rU   rþ  r–   )r+   r,   s   "€r-   r.   rK    s¨   ø€ ÷ Dñ Dá×'Ñ'ðDñ !×+Ñ+¨dÕ2ðDñ !&× 0Ñ 0°4Õ 7ð	Dñ
 ×#Ñ# dÕ*ðDñ ×(Ñ(¨4Õ/ðDñ ×(Ñ(¨4Õ/ðDñ #'ðDñ 
×	Ñ	ñDr0   c                óŒ  € \        V R4      '       d   V P                  4        VP                  ^ ,          p	V P                  VVVVRR7      p
V
P                  pVf¸   Vfž   V P
                  P                  .V P
                  P                  ,          pWÀP
                  P                  P                  .,           p\        P                  ! V.\        P                  VP                  R7      pVP                  V	^4      pV P                  4       ! V4      pVf   \        P                   ! V4      pVP#                  VP                  VP$                  4      pV P'                  WFR7      pVP)                  Wë4      pRVRV/pV P*                  P
                  P,                  '       g   WOR&   V P*                  P.                  ! R	/ VBVB pV# )
aþ  
Overrides `generate` function to be able to use the model as a conditional generator.

Args:
    pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)):
        Input images to be processed.
    qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt to be fed to the Q-Former module.
    qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt for the generation.
    attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Embedded representation of the inputs. Should be float, not int tokens.
    interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
        Whether to interpolate the positional encoding of the image embeddings.

Returns:
    captions (list): A list of strings of length batch_size * num_captions.
r  Trl  r(  r:  rF  r­   ré  r1   )rg  r  r„   rd  rW  rZ   Úimage_token_indexr  r  Úbos_token_idrT   rÔ   r,  rÂ  Úrepeatr^  r<  rš   r˜   r1  r=  r  Úis_encoder_decoderÚgenerate)rK   r•   r5  r6  ré  r­   rF  r’   Úgenerate_kwargsrž   rc  rD  Úimage_tokensÚstart_tokensr0  ÚinputsrE  s   &&&&&&&&,        r-   ru  Ú-InstructBlipForConditionalGeneration.generate  s™  € ôD �4˜×)Ò)à×'Ñ'Ô)à!×'Ñ'¨Õ*ˆ
ØBF×BYÑBYØØ/Ø#9Ø%=Øð CZó C
ˆð !/× <Ñ <ÐàÒ ØÒ Ø $§¡× =Ñ =Ð>ÀÇÁ×A]ÑA]Õ]�Ø+¯{©{×/FÑ/F×/SÑ/SÐ.TÕT�Ü!ŸLšL¨,¨¼u¿z¹zÐR^×ReÑReÔf�	Ø%×,Ñ,¨Z¸Ó;�	Ø ×5Ñ5Ô7¸	ÓBˆMàÒ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ó^ÐØ%×4Ñ4Ð5GÓ_ˆà! =Ð2BÀNÐSˆØ×"Ñ"×)Ñ)×<×<Ð<Ø"+�;Ñà×%Ñ%×.Ò.ÑK°ÐK¸?ÑKˆàˆr0   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rY   rý   )r+   r,   s   "€r-   r.   rK  ù  s   ø‡ ‚ ñ Ñò r0   rG  rë   ra  )NNNNNNNF)NNNNNF)r2   r3   r4   r5   rb  r2  rH  rd   rP  rU  rY  r\  r  r   r   rd  r1  r£   rT   r4  ru  r7   r8   r9   r¦   r§   s   @@r-   r  r  ù  sÂ   ù‡ € ð %€Oà!ÐØ+Ð,Ð÷ó ò(B÷;ð ;÷:ò1ò?ð( Ø÷@ñ @ó ó ð@÷D"ð "ð Ø÷B
ñ B
ó ó ðB
ðH ‡]‚]ƒ_÷Dñ Dó ðD÷E	 … r0   r  )rÞ   )rï  r  r  r  rJ  )Rr6   Úcollections.abcr   Údataclassesr   Útypingr   rT   r   Ú r   r  Úactivationsr   Ú
generationr   Úmasking_utilsr	   Úmodeling_flash_attention_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úautor   r    r!   Úconfiguration_instructblipr"   r#   r$   Ú
get_loggerr2   r  r&   r=   r°   rW   r¼   r¾   ré   rû   r  r6  rJ  r)  r*  r’  rž  r©  r(  rÐ  r  rï  r  r  Ú__all__r1   r0   r-   Ú<module>r�     s<  ðñ "å $Ý !Ý ã Ý å &Ý !Ý )Ý 6Ý BÝ 9÷÷ ÷ GÝ &Ý 6ß j× jÝ 7ß Eß IÑ Iß oÑ oð 
×	Ò	˜HÓ	%€ð Ø
ô	PÐ.Hó 	Pó ó ð	Pñ ðôð
 ô
°kó 
ó óð
ô<G 2§9¡9ô G÷V%ô0G)˜BŸI™Iô G)ôV�b—i‘iô ô Ð9ô ð@ ô i /ó  ió ð iôH@˜"Ÿ)™)ô @ô@3Ð9ô 3ôlE)¨B¯I©Iô E)ôR B§I¡Iô ô  2§9¡9ô  ô4 b§i¡iô ô  §	¡	ô ôRÐ9ô Rôl!
 §¡ô !
ôH- B§I¡Iô -ô`a
Ð:ô a
ñH ðôô
U
Ð3ó U
óð
U
ñp ðôô]Ð+FÈó ]óð]ò@‚r0   