+
    œ°³jéŒ  ã                   ó"  € ^ RI Ht ^ RIt^ RIHt ^RIHt ^RIHtH	t	 ^RI
Ht ^RIHtHt ^RIHtHtHtHt ^R	IHt ^R
IHt ^RIHtHtHtHt ^RIHt ^RIH t H!t! ^RI"H#t#H$t$H%t%H&t&H't'H(t(H)t)H*t*H+t+H,t, ^RI-H.t.H/t/H0t0H1t1H2t2H3t3H4t4 ^RI5H6t6H7t7 ]Pp                  ! ]94      t:RGR R llt; ! R R]$4      t< ! R R]*4      t= ! R R]+4      t> ! R R],4      t? ! R R]#4      t@ ! R R]&4      tA ! R R ](4      tB ! R! R"]'4      tC ! R# R$]%4      tD ! R% R&])4      tE] ! R' R(]4      4       tF ! R) R*]F4      tG ! R+ R,]P�                  4      tI ! R- R.]P�                  4      tJ ! R/ R0]P�                  4      tK]] ! R1 R2]4      4       4       tL ! R3 R4]P�                  4      tM ! R5 R6]P�                  4      tN ! R7 R8]24      tO ! R9 R:]34      tP ! R; R<]04      tQ ! R= R>].4      tR ! R? R@]/4      tS ! RA RB]14      tT ! RC RD]T4      tU ! RE RF]T]4      tV. RHOtWR# )Ié    )Ú	dataclassN)Únn)Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)
ÚEsmAttentionÚEsmEmbeddingsÚ
EsmEncoderÚEsmIntermediateÚEsmLayerÚ	EsmOutputÚ	EsmPoolerÚEsmRotaryEmbeddingÚEsmSelfAttentionÚEsmSelfOutput)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚ	repeat_kv)ÚEvollaConfigÚSaProtConfigc                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
é   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingÚdropoutÚkwargs)r   ÚModuleÚtorchÚTensorÚfloatr   r   )Úformats   "Új/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/evolla/modular_evolla.pyÚ__annotate__r;   C   s{   € ÷  %ñ  %Ü�I‰Ið %ä�<‰<ð %ô 
�‰ð %ô �<‰<ð	 %ô
 —L‘L 4Õ'ð %ô �T�\ð %ô ð %ô Ô'Õ(ñ %ó    c                 óü  € Vf   VP                  R4      R,          p\        V R^4      p\        W(4      p\        W84      p\        P                  ! WP                  ^^4      4      V,          p	Ve	   W”,           p	\        P                  P                  V	RR7      p	\        P                  P                  W–V P                  R7      p	\        P                  ! W“4      p
V
P                  ^^4      P                  4       p
W©3# )NÚnum_key_value_groups©Údim)ÚpÚtrainingéÿÿÿÿç      à¿)ÚsizeÚgetattrr(   r6   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr3   rB   Ú
contiguous)r-   r.   r/   r0   r1   r2   r3   r4   Ún_repÚattn_weightsÚattn_outputs   &&&&&&&,   r:   Úeager_attention_forwardrO   C   sÖ   € ð ‚Ø—*‘*˜R“. DÕ(ˆô
 �FÐ2°AÓ6€EÜ
�CÓ
€CÜ�eÓ#€Eô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r<   c                   ó2   a a€ ] tR t^ft oV 3R ltRtVtV ;t# )ÚEvollaSaProtEmbeddingsc                ó*   <€ \         SV `  V4       V =R # ©N)ÚsuperÚ__init__Úposition_ids©ÚselfÚconfigÚ	__class__s   &&€r:   rU   ÚEvollaSaProtEmbeddings.__init__g   s   ø€ Ü‰Ñ˜Ô àÒr<   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rU   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rZ   Ú__classdict__s   @@r:   rQ   rQ   f   s   ù‡ € ÷õ r<   rQ   c                   ó^   a a€ ] tR t^mt oRV3R lV 3R llltRV3R lV 3R llltRtVtV ;t# )ÚEvollaSaProtRotaryEmbeddingc                ó    <€ V ^8„  d   QhRS[ /# ©r,   rY   ©r*   )r9   re   s   "€r:   r;   Ú(EvollaSaProtRotaryEmbedding.__annotate__n   s   ø€ ÷ !ñ !™|ñ !r<   c                ó&   <€ \         SV `  V4       R # rS   )rT   rU   )rX   rY   ÚdevicerZ   s   &&&€r:   rU   Ú$EvollaSaProtRotaryEmbedding.__init__n   s   ø€ Ü‰Ñ˜Ö r<   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# )r,   rY   Úreturn)r*   Útupler6   r7   r8   )r9   re   s   "€r:   r;   rk   q   s.   ø€ ÷ ?ñ ?±ð ?ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ ?r<   c                ó"   <€ \         SV `  V 4      # rS   )rT   Úcompute_default_rope_parameters)rY   rm   r4   rZ   s   &&,€r:   rs   Ú;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s   ø€ Ü‰wÑ6°vÓ>Ð>r<   r\   rS   )	r]   r^   r_   r`   rU   rs   ra   rb   rc   rd   s   @@r:   rg   rg   m   s   ù‡ € ÷!õ !÷?÷ ?õ ?r<   rg   c                   ó   € ] tR t^utRtR# )ÚEvollaSaProtSelfAttentionr\   N©r]   r^   r_   r`   ra   r\   r<   r:   rv   rv   u   ó   † Ûr<   rv   c                   ó   € ] tR t^ytRtR# )ÚEvollaSaProtSelfOutputr\   Nrw   r\   r<   r:   rz   rz   y   rx   r<   rz   c                   ó   € ] tR t^}tRtR# )ÚEvollaSaProtAttentionr\   Nrw   r\   r<   r:   r|   r|   }   rx   r<   r|   c                   ó   € ] tR t^�tRtR# )ÚEvollaSaProtIntermediater\   Nrw   r\   r<   r:   r~   r~   �   rx   r<   r~   c                   ó   € ] tR t^…tRtR# )ÚEvollaSaProtOutputr\   Nrw   r\   r<   r:   r€   r€   …   rx   r<   r€   c                   ó   € ] tR t^‰tRtR# )ÚEvollaSaProtLayerr\   Nrw   r\   r<   r:   r‚   r‚   ‰   rx   r<   r‚   c                   ó   € ] tR t^�tRtR# )ÚEvollaSaProtEncoderr\   Nrw   r\   r<   r:   r„   r„   �   rx   r<   r„   c                   ó   € ] tR t^‘tRtR# )ÚEvollaSaProtPoolerr\   Nrw   r\   r<   r:   r†   r†   ‘   rx   r<   r†   c                   ó´   a a€ ] tR t^•t oR.tRtRtRtRtR]	R]
! ]^RR7      .R]
! ]^RR7      ./t]P                  ! 4       V 3R	 l4       tV3R
 ltRtVtV ;t# )ÚEvollaSaProtPreTrainedModelr‚   TÚhidden_statesÚ
attentionsÚ	attention)ÚindexÚ
layer_nameÚcross_attentionsÚcrossattentionc                óÒ   <€ \         SV `  V4       \        V\        4      '       dA   VP	                  VP
                  4      w  r#\        P                  ! \        VR 4      V4       R# R# )Úinv_freqN)	rT   Ú_init_weightsÚ
isinstancerg   rs   rY   ÚinitÚcopy_rF   )rX   r-   Úcurr_inv_freqÚ_rZ   s   &&  €r:   r’   Ú)EvollaSaProtPreTrainedModel._init_weights¦   sO   ø€ ä‰Ñ˜fÔ%Ü�fÔ9×:Ò:Ø%×EÑEÀfÇmÁmÓTÑˆMÜ�JŠJ”w˜v zÓ2°MÖBñ ;r<   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ri   rj   )r9   re   s   "€r:   r;   Ú(EvollaSaProtPreTrainedModel.__annotate__•   s   ø‡ ‚ áÑò r<   r\   )r]   r^   r_   r`   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr‚   r   rv   Ú_can_record_outputsr6   Úno_gradr’   Ú__annotate_func__ra   rb   rc   rd   s   @@r:   rˆ   rˆ   •   sƒ   ù‡ € ð -Ð-ÐØÐØ€NØÐØ"&Ðð 	Ð*Ø‘~Ð&?ÀqÐU`ÔaÐbØÙÐ4¸AÐJZÔ[ð
ðÐð ‡]‚]ƒ_ôCó ðC÷% … r<   rˆ   c                   ót   a a€ ] tR t^®t oV3R lV 3R lltR tR t]]RV3R lR ll4       4       t	Rt
VtV ;t# )	ÚEvollaSaProtProteinEncoderc                ó    <€ V ^8„  d   QhRS[ /# ri   rj   )r9   re   s   "€r:   r;   Ú'EvollaSaProtProteinEncoder.__annotate__¯   s   ø€ ÷ ñ ™|ñ r<   c                ó¨   <€ \         SV `  V4       \        V4      V n        \	        VR 7      V n        \        V4      V n        V P                  4        R# ©©rY   N)	rT   rU   rQ   Ú
embeddingsrg   Úrotary_embeddingsr„   ÚencoderÚ	post_initrW   s   &&€r:   rU   Ú#EvollaSaProtProteinEncoder.__init__¯   s?   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆŒÜ!<ÀFÔ!KˆÔÜ*¨6Ó2ˆŒØ�‰Ör<   c                ó.   € V P                   P                  # rS   ©rª   Úword_embeddings©rX   s   &r:   Úget_input_embeddingsÚ/EvollaSaProtProteinEncoder.get_input_embeddings¶   s   € Ø�‰×.Ñ.Ð.r<   c                ó&   € WP                   n        R # rS   r°   ©rX   r0   s   &&r:   Úset_input_embeddingsÚ/EvollaSaProtProteinEncoder.set_input_embeddings¹   s   € Ø*/�‰Ö'r<   c                ó¤   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          S[,          /# )r,   Ú	input_idsNr1   rp   )r6   r7   rq   r   )r9   re   s   "€r:   r;   r¦   ¾   sJ   ø€ ÷ !
ñ !
á—<‘< $Õ&ð!
ñ Ÿ™ tÕ+ð!
ñ
 
‰u�|‰|Õ	ÑKÕ	Kñ!
r<   c                óØ  € VP                  4       pVw  rVVP                  pVf   \        P                  ! WV3VR7      pV P	                  WR7      p\        V P                  VVR7      p\        P                  ! WgR7      P                  ^ 4      p	V P                  W‰4      p
V P                  ! V3RVRV
/VB pV^ ,          p\        VVP                  VP                  VP                  R7      # )N©rm   ©rº   r1   )rY   Úinputs_embedsr1   r1   Úposition_embeddings)Úlast_hidden_stater‰   rŠ   rŽ   )rE   rm   r6   Úonesrª   r	   rY   ÚarangeÚ	unsqueezer«   r¬   r   r‰   rŠ   rŽ   )rX   rº   r1   r4   Úinput_shapeÚ
batch_sizeÚ
seq_lengthrm   r¾   rV   r¿   Úencoder_outputsÚsequence_outputs   &&&,         r:   ÚforwardÚ"EvollaSaProtProteinEncoder.forward¼   sï   € ð  —n‘nÓ&ˆØ!,Ñˆ
à×!Ñ!ˆØÒ!Ü"ŸZšZ¨*Ð)AÈ6ÔRˆNØŸ™°)˜Ó[ˆä2Ø—;‘;Ø'Ø)ô
ˆô —|’| JÔ>×HÑHÈÓKˆØ"×4Ñ4°]ÓQÐàŸ,š,Øñ
Ø*8ð
ØNað
Øekñ
ˆð *¨!Õ,ˆä;Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r<   )rª   r¬   r«   rS   )r]   r^   r_   r`   rU   r³   r·   r   r   rÉ   ra   rb   rc   rd   s   @@r:   r¤   r¤   ®   s?   ù‡ € ÷ó ò/ò0ð  Ø÷!
ñ !
ó ó  ÷!
ð !
r<   r¤   c                   ó<   a a€ ] tR t^ât oRV 3R lltR tRtVtV ;t# )Ú!EvollaSequenceCompressorAttentionc                ó„  <€ \         SV `  4        VR,          V n        W0n        W#,          p\        P
                  ! V4      V n        \        P
                  ! V4      V n        \        P                  ! WRR7      V n	        \        P                  ! W^,          RR7      V n
        \        P                  ! WARR7      V n        R# )ç      à?F©ÚbiasNrD   )rT   rU   ÚscaleÚheadsr   Ú	LayerNormÚ
norm_mediaÚnorm_latentsÚLinearÚto_qÚto_kvÚto_out)rX   r@   Údim_headrÒ   Ú	inner_dimrZ   s   &&&& €r:   rU   Ú*EvollaSequenceCompressorAttention.__init__ã   s�   ø€ Ü‰ÑÔØ˜t•^ˆŒ
ØŒ
ØÕ$ˆ	äŸ,š, sÓ+ˆŒÜŸLšL¨Ó-ˆÔä—I’I˜c°5Ô9ˆŒ	Ü—Y’Y˜s°¥M¸Ô>ˆŒ
Ü—i’i 	°UÔ;ˆŽr<   c                óN  € V P                  V4      pV P                  V4      pV P                  pV P                  V4      p\        P
                  ! W3RR7      pV P                  V4      P                  ^RR7      w  rxVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pWPP                  ,          p\        P                  ! WWP                  RR4      4      p	W™P                  RRR7      P                  4       ,
          p	V	P                   w  r«rÍ\        P"                  ! W¼4      P%                  VP&                  4      pVR,          pVR,          pVV,          pV	P)                  ^V,
          P+                  4       R4      p	V	P-                  RR7      p\        P                  ! VV4      pVP                  ^ ^^^4      pVP/                  VP                  ^ 4      VP                  ^4      R4      pV P1                  V4      # )	z¤
Args:
    x (torch.Tensor): image features
        shape (b, n1, D)
    latent (torch.Tensor): latent features
        shape (b, n2, D);  n2: num of latent tokens
r?   T©r@   ÚkeepdiméþÿÿÿrC   ©ºNNNNNrâ   )Nrâ   râ   Ng     ˆÃÀ)rÔ   rÕ   rÒ   r×   r6   ÚcatrØ   ÚchunkÚviewrE   ÚpermuterÑ   rG   rH   ÚamaxÚdetachÚshaperÁ   Útorm   Úmasked_fillÚboolrJ   ÚreshaperÙ   )rX   ÚxÚlatentsÚmaskÚhÚqÚkv_inputÚkÚvÚsimÚbsÚnhÚskdÚokdrÁ   Úmask_expÚones_expÚattnÚouts   &&&&               r:   rÉ   Ú)EvollaSequenceCompressorAttention.forwardð   s"  € ð �O‰O˜AÓˆØ×#Ñ# GÓ,ˆà�J‰Jˆà�I‰I�gÓˆÜ—9’9˜a˜\¨rÔ2ˆØ�z‰z˜(Ó#×)Ñ)Ø�2ð *ó 
‰ˆð �F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ—
‘
�Nˆô �lŠl˜1Ÿk™k¨"¨bÓ1Ó2ˆØ—H‘H ¨T�HÓ2×9Ñ9Ó;Õ;ˆØŸ9™9Ñˆ�Ü�zŠz˜"Ó"×%Ñ% d§k¡kÓ2ˆØÐ(Õ)ˆØÐ(Õ)ˆØ˜(Õ"ˆà�o‰o˜q 4�xŸo™oÓ/°Ó6ˆØ�{‰{˜rˆ{Ó"ˆÜ�lŠl˜4 Ó#ˆØ�k‰k˜!˜Q  1Ó%ˆð �k‰k˜#Ÿ(™( 1›+ s§x¡x°£{°BÓ7ˆà�{‰{˜3ÓÐr<   )rÒ   rÕ   rÔ   rÑ   rØ   rÙ   r×   )é@   é   ©	r]   r^   r_   r`   rU   rÉ   ra   rb   rc   rd   s   @@r:   rÌ   rÌ   â   s   ù‡ € ÷<÷) ò ) r<   rÌ   c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚEvollaFeedForwardi  c                ó$  <€ \         SV `  4        \        W,          4      p\        P                  ! V4      V n        \        P                  ! WR R7      V n        \        P                  ! 4       V n	        \        P                  ! W1R R7      V n
        R# ©FrÏ   N)rT   rU   Úintr   rÓ   ÚnormrÖ   Úfc1ÚGELUÚ
activationÚfc2)rX   r@   ÚmultrÛ   rZ   s   &&& €r:   rU   ÚEvollaFeedForward.__init__  sZ   ø€ Ü‰ÑÔÜ˜�
“Oˆ	ä—L’L Ó%ˆŒ	Ü—9’9˜S°%Ô8ˆŒÜŸ'š'›)ˆŒÜ—9’9˜Y°%Ô8ˆŽr<   c           	     ó~   € V P                  V P                  V P                  V P                  V4      4      4      4      # rS   )r  r  r	  r  )rX   rî   s   &&r:   rÉ   ÚEvollaFeedForward.forward&  s+   € Ø�x‰x˜Ÿ™¨¯©°·±¸1³Ó(>Ó?Ó@Ð@r<   )r  r	  r  r  )é   r  rd   s   @@r:   r  r    s   ù‡ € ÷9÷Aò Ar<   r  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )Ú!EvollaSequenceCompressorResampleri*  c                ó    <€ V ^8„  d   QhRS[ /# ri   ©r)   )r9   re   s   "€r:   r;   Ú.EvollaSequenceCompressorResampler.__annotate__+  s   ø€ ÷ Qñ Q™|ñ Qr<   c                ó´  <€ \         SV `  4        VP                  P                  pVP                  V n        \        P                  ! \        P                  ! V P
                  V4      R R7      V n
        \        P                  ! . 4      V n        \        VP                  4       Fg  pV P                  P                  \        P                  ! \!        W!P"                  VP$                  R7      \'        W!P(                  R7      .4      4       Ki  	  \        P*                  ! VP                  4      V n        \        P.                  ! W!P                  4      V n        R# )T)Úrequires_grad)r@   rÚ   rÒ   )r@   r  N)rT   rU   Úprotein_encoder_configÚhidden_sizeÚresampler_num_latentsÚnum_latentsr   Ú	Parameterr6   Úrandnrï   Ú
ModuleListÚlayersÚrangeÚresampler_depthÚappendrÌ   Úresampler_dim_headÚresampler_headsr  Úresampler_ff_multrÓ   r  rÖ   Úprotein_projector)rX   rY   Úprotein_repr_dimr—   rZ   s   &&  €r:   rU   Ú*EvollaSequenceCompressorResampler.__init__+  sö   ø€ Ü‰ÑÔØ!×8Ñ8×DÑDÐØ!×7Ñ7ˆÔÜ—|’|¤E§K¢K°×0@Ñ0@ÐBRÓ$SÐcgÔhˆŒÜ—m’m BÓ'ˆŒÜ�v×-Ñ-Ö.ˆAØ�K‰K×ÑÜ—’ä9Ø 0×;TÑ;TÐ\b×\rÑ\rôô *Ð.>×E]ÑE]Ô^ð	óö	ñ /ô —L’L ×!3Ñ!3Ó4ˆŒ	Ü!#§¢Ð+;×=OÑ=OÓ!PˆÖr<   c                ó’  € VP                   ^ ,          pVP                   w  rE\        P                  ! W@P                  4      P	                  VP
                  4      p\        P                  ! W&3^R7      p\        P                  ! V4      P	                  V P                  P
                  4      pV P                  R,          VP                  R^^4      ,          pVP	                  VP                  4      pV P                   F$  w  ršV	! WV4      V,           pV
! V4      V,           pK&  	  V P                  V4      pV P                  V4      # )r   r?   NrC   )ré   r6   rÁ   r  rê   rm   rã   rï   rå   Údtyper   r'  r  )rX   Úembedsrð   Úbr÷   r—   Úlatent_maskrÁ   rï   rý   ÚffÚtransformed_features   &&&         r:   rÉ   Ú)EvollaSequenceCompressorResampler.forward@  sø   € Ø�L‰L˜�Oˆà—
‘
‰ˆÜ—j’j ×%5Ñ%5Ó6×9Ñ9¸$¿+¹+ÓFˆÜ�yŠy˜$Ð,°!Ô4ˆô �zŠz˜!‹}×Ñ §¡× 3Ñ 3Ó4ˆØ—,‘,˜tÕ$ t§y¡y°°Q¸Ó':Õ:ˆØ—*‘*˜VŸ\™\Ó*ˆØŸœ‰HˆDÙ˜6¨DÓ1°GÕ;ˆGÙ˜“k GÕ+ŠGñ $ð #×4Ñ4°WÓ=Ðà�y‰yÐ,Ó-Ð-r<   )rï   r   r  r  r'  r  rd   s   @@r:   r  r  *  s   ù‡ € ÷Qó Q÷*.ò .r<   r  c                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚEvollaProteinEncoderModelOutputiT  a>  
sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
    Compressed sequence representation produced by the sequence compressor module. The sequence length is
    reduced from the original input length to `compressed_seq_len` via learned compression.
Nc                óú   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )r,   NÚsequence_compressor_outputrÀ   .r‰   rŠ   )r6   ÚFloatTensorrq   )r9   re   s   "€r:   r;   Ú,EvollaProteinEncoderModelOutput.__annotate__T  sy   ø‡ ‚ ñ !&× 1Ñ 1°DÕ 8Ñ?ñ ñ ×(Ñ(¨4Õ/Ñ6ñ ñ ™×*Ñ*¨CÐ/Õ0°4Õ7Ñ>ñ ñ ‘e×'Ñ'¨Ð,Õ-°Õ4Ñ;ò r<   r\   )r]   r^   r_   r`   Ú__doc__r5  rÀ   r‰   rŠ   r¢   ra   rb   ©re   s   @r:   r3  r3  T  s*   ø‡ € ñð <@ÐØ26ÐØ:>€MØ7;€J÷ ƒ r<   r3  c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )ÚEvollaProteinEncoderic  c                ó    <€ V ^8„  d   QhRS[ /# ri   r  )r9   re   s   "€r:   r;   Ú!EvollaProteinEncoder.__annotate__d  s   ø€ ÷ ^ñ ^™|ñ ^r<   c                ó|   <€ \         SV `  4        \        VP                  R 7      V n        \        VR 7      V n        R# r¨   )rT   rU   r¤   r  Úmodelr  Úsequence_compressor_resamplerrW   s   &&€r:   rU   ÚEvollaProteinEncoder.__init__d  s.   ø€ Ü‰ÑÔÜ/°v×7TÑ7TÔUˆŒ
Ü-NÐV\Ô-]ˆÖ*r<   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r,   rº   r1   )r6   Ú
LongTensorr6  )r9   re   s   "€r:   r;   r=  j  s'   ø€ ÷ 
ñ 
¡×!1Ñ!1ð 
Á5×CTÑCTñ 
r<   c                óŽ   € V P                  WR 7      pVP                  pV P                  WR4      p\        VVP                  R7      # )r½   )r5  rÀ   )r?  rÀ   r@  r3  )rX   rº   r1   r4   Úprotein_outputÚprotein_embedsÚsequence_reprs   &&&,   r:   rÉ   ÚEvollaProteinEncoder.forwardi  sF   € àŸ™¨i˜ÓWˆØ'×9Ñ9ˆØ×:Ñ:¸>ÓZˆä.Ø'4Ø,×>Ñ>ô
ð 	
r<   )r?  r@  )
r]   r^   r_   r`   rU   r   rÉ   ra   rb   rc   rd   s   @@r:   r;  r;  c  s*   ù‡ € ÷^ó ^ð
 ÷
ó ÷
ð 
r<   r;  c                   óR   a a€ ] tR tRt oRV3R lV 3R llltR tRR ltRtVtV ;t	# )	Ú#EvollaSequenceAlignerCrossAttentioniu  c                óV   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[ R,          /# )r,   Úprotein_encoder_dimNÚstructure_encoder_dimÚmsa_encoder_dim)r  )r9   re   s   "€r:   r;   Ú0EvollaSequenceAlignerCrossAttention.__annotate__v  s7   ø€ ÷ 1:ñ 1:ñ ! 4�Zð1:ñ  # T�zð	1:ñ
 ˜t�ñ1:r<   c                óÌ  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  R,          V n        \        V P                  V P                  ,          4      V n        V P                  V P                  ,          V n        VP                  pVP                  pVP                  p\        P                  ! V P                  V P                  4      V n        VeL   \        P                  ! W P                  4      V n        \        P                  ! W P                  4      V n        MRV n        RV n        VeL   \        P                  ! W0P                  4      V n        \        P                  ! W0P                  4      V n        MRV n        RV n        VeL   \        P                  ! W@P                  4      V n        \        P                  ! W@P                  4      V n        MRV n        RV n        \)        V P                  4      V n        \        P,                  ! V4      V n        \        P                  ! V P                  V P                  VR7      V n        \3        V P                  V4      V n        \        P6                  ! \8        P:                  ! R.4      4      V n        \        P6                  ! \8        P:                  ! R.4      4      V n        R# )rÎ   NrÏ   ç        rD   ) rT   rU   r  Únum_attention_headsrÑ   r  Úattention_head_sizeÚall_head_sizeÚ$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÖ   r.   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normÚDropoutr3   Úout_projr  r/  r  r6   ÚtensorÚgate_attentionÚgate_ffw)	rX   rY   rL  rM  rN  Úattention_probs_dropout_probÚenable_biasÚffn_multrZ   s	   &&&&&   €r:   rU   Ú,EvollaSequenceAlignerCrossAttention.__init__v  s  ø€ ô 	‰ÑÔà!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×-Ñ-¨tÕ3ˆŒ
Ü#& t×'7Ñ'7¸$×:RÑ:RÕ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔà'-×'RÑ'RÐ$Ø×0Ñ0ˆØ×*Ñ*ˆä—Y’Y˜t×/Ñ/°×1CÑ1CÓDˆŒ
ØÒ*Ü!ŸyšyÐ)<×>PÑ>PÓQˆDÔÜ!#§¢Ð+>×@RÑ@RÓ!SˆDÕà#ˆDÔØ!%ˆDÔà Ò,Ü!#§¢Ð+@×BTÑBTÓ!UˆDÔÜ#%§9¢9Ð-B×DVÑDVÓ#WˆDÕ à!%ˆDÔØ#'ˆDÔ àÒ&ÜŸ9š9 _×6HÑ6HÓIˆDŒLÜŸYšY ×8JÑ8JÓKˆD�NàˆDŒLØ!ˆDŒNä+¨D×,<Ñ,<Ó=ˆÔä—z’zÐ">Ó?ˆŒäŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ;ÔWˆŒä# D×$4Ñ$4°hÓ?ˆŒÜ Ÿlšl¬5¯<ª<¸¸Ó+>Ó?ˆÔÜŸš¤U§\¢\°3°%Ó%8Ó9ˆŽr<   c	                ó	  € WgV.p	V	 U
u. uF
  qªf   K  V
NK  	  p	p
V	'       g   \        R4      h\        P                  ! V	^R7      p	V P                  V4      pV P	                  V4      pV P
                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpV P                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpV P                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpWÎV.pV U
u. uF
  qªf   K  V
NK  	  pp
\        P                  ! V^R7      pWßV.pV U
u. uF
  qªf   K  V
NK  	  pp
\        P                  ! V^R7      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pW°P"                  ,          pVfO   \        P$                  ! VP                  ^ 4      VP                  ^4      4      P                  VP&                  4      pVR,          V	R,          ,          p\        P(                  ! VVP+                  RR	4      4      pVVP-                  RRR7      P/                  4       ,
          pVP1                  ^V,
          P3                  4       \        P4                  ! VP6                  4      P8                  4      p\:        P<                  ! RR7      ! V4      p\        P(                  ! VV4      pVP!                  ^ ^^^4      P?                  4       pVP                  4       RR	 V P@                  3,           pVP                  ! V!  pV PC                  V4      pV# u up
i u up
i u up
i )
zÁ
query_states: text
key_value_states: protein
query_states: [bs, query_seq_len, dim]
key_value_states: [bs, kv_seq_len, dim]
query_attn_mask: [bs, query_seq_len]
kv_attn_mask: [bs, kv_seq_len]
Nz=At least one modality should be provided for cross attention.r?   TrÞ   rC   )râ   Nrâ   Nrá   rà   )"Ú
ValueErrorr6   rã   r_  r.   rX  rY  rê   rZ  r[  r\  r]  rE   rR  rS  rå   ræ   rÑ   rÁ   rm   rG   rH   rç   rè   rë   rì   Úfinfor+  Úminr   ÚSoftmaxrK   rT  ra  )rX   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskr—   Úquery_layerÚkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaÚ	key_layerÚvalue_layerÚnew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper1   rM   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes   &&&&&&&&&                    r:   Úcross_attentionÚ3EvollaSequenceAlignerCrossAttention.cross_attention©  s?  € ð* -ÐFVÐWˆÙ#/ÓA¡<˜aŸ˜¡<ˆÐAßÜÐ\Ó]Ð]Ü—y’y °1Ô5ˆà×)Ñ)¨,Ó7ˆð —j‘j Ó-ˆà×ÑÒ'¨D×,>Ñ,>Ò,JØ'?×'BÑ'BÀ<Ó'PÐ$Ø $× 0Ñ 0Ð1IÓ JÐØ"&×"4Ñ"4Ð5MÓ"NÑà $ÐØ"&Ðà×ÑÒ)¨d×.BÑ.BÒ.NØ)C×)FÑ)FÀ|Ó)TÐ&Ø"&×"4Ñ"4Ð5OÓ"PÐØ$(×$8Ñ$8Ð9SÓ$TÑ!à"&ÐØ$(Ð!à�<‰<Ò#¨¯©Ò(BØ#7×#:Ñ#:¸<Ó#HÐ Ø ŸL™LÐ)=Ó>ˆMØ"Ÿn™nÐ-AÓB‰Oà ˆMØ"ˆOà&¸]ÐKˆ	Ù )Ó;¡	˜1—Q�Q¡	ˆ	Ð;Ü—I’I˜i¨QÔ/ˆ	à*À?ÐSˆÙ"-Ó?¡+˜Q—q�q¡+ˆÐ?Ü—i’i °Ô3ˆà +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
õ !
Ðð "×&Ò&Ð(=Ñ>×FÑFÀqÈ!ÈQÐPQÓRˆà'Ÿn™nÓ.¨s°Ð3Ø×$Ñ$Ø×$Ñ$ð7
õ 
Ðð —N’NÐ$7Ñ8×@Ñ@ÀÀAÀqÈ!ÓLˆ	à +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
õ !
Ðð "×&Ò&Ð(=Ñ>×FÑFÀqÈ!ÈQÐPQÓRˆà!§J¡JÕ.ˆð Ò"Ü#Ÿjšj¨×):Ñ):¸1Ó)=¸|×?PÑ?PÐQRÓ?SÓT×WÑWÐXd×XkÑXkÓlˆOØ(Ð)9Õ:¸\ÐJZÕ=[Õ[ˆä—|’| K°×1DÑ1DÀRÈÓ1LÓMˆØ# l×&7Ñ&7¸BÈÐ&7Ó&M×&TÑ&TÓ&VÕVˆØ'×3Ñ3Ø�Õ×%Ñ%Ó'¬¯ª°\×5GÑ5GÓ)H×)LÑ)Ló
Ðô Ÿ*š*¨Õ,Ð-=Ó>ˆô Ÿš _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÕ"SÐØ%×*Ò*Ð,CÑDˆàŸ™ mÓ4ˆàÐùòq BùòL <ùò @s"   ‰Q5“Q5Å,Q:Å6Q:ÆQ?Æ(Q?c                óâ  € Ve   VP                   w  rÞpVfk   \        P                  ! WÞ4      P                  V	P                  4      V	P                  Wí3R7      P                  ,          P                  VP                  4      pMR pVe‚   VP                   w  pppVfm   \        P                  ! VV4      P                  V	P                  4      V
P                  VV3R7      P                  ,          P                  VP                  4      pMR pVe‚   VP                   w  pppVfm   \        P                  ! VV4      P                  V	P                  4      VP                  VV3R7      P                  ,          P                  VP                  4      pMR pTpVe   VP                  4       '       g5   Ve   VP                  4       '       g   Ve£   VP                  4       '       d�   TpV P                  VVVVVVVVR7      p\        P                  ! V P                  4      V,          pVV,           pTpV P                  V4      \        P                  ! V P                  4      ,          pVV,           pV# )N)rE   )rn  ro  rp  rq  rr  rs  rt  ru  )ré   r6   rÁ   rê   rm   ÚexpandÚTÚanyr‡  Útanhrc  r/  rd  )rX   rn  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrr  rs  rt  ru  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesr÷   Úprotein_kv_seq_lenr@   Ústructure_kv_seq_lenÚmsa_kv_seq_lenr‰   Úresiduals   &&&&&&&&&&&&&       r:   rÉ   Ú+EvollaSequenceAlignerCrossAttention.forward  sP  € ð Ò(Ø*;×*AÑ*AÑ'ˆB CØ#Ò+ä—J’J˜rÓ6×9Ñ9Ð:L×:SÑ:SÓTØ(×/Ñ/Ð6HÐ5MÐ/ÓN×PÑPõQç‘"Ð&×-Ñ-Ó.ð %øð
 $(Ð àÒ*Ø,?×,EÑ,EÑ)ˆBÐ$ cØ%Ò-ä—J’J˜rÐ#7Ó8×;Ñ;Ð<N×<UÑ<UÓVØ*×1Ñ1Ð8LÈbÐ7QÐ1ÓR×TÑTõUç‘"Ð(×/Ñ/Ó0ð 'øð
 &*Ð"àÒ$Ø&3×&9Ñ&9Ñ#ˆB� ØÒ'ä—J’J˜r >Ó2×5Ñ5Ð6H×6OÑ6OÓPØ$×+Ñ+°.À"Ð1EÐ+ÓF×HÑHõIç‘"�]×)Ñ)Ó*ð !øð
  $ÐØ$ˆð Ò*Ð/C×/GÑ/G×/IÒ/IØ#Ò/Ð4J×4NÑ4N×4PÒ4PØÒ)Ð.>×.BÑ.B×.DÒ.Dà$ˆHØ ×0Ñ0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ó 	ˆMô "ŸJšJ t×':Ñ':Ó;¸mÕKˆMà$ }Õ4ˆMà$ˆHØ ŸG™G MÓ2´U·Z²ZÀÇÁÓ5NÕNˆMØ$ }Õ4ˆMàÐr<   )rT  rS  r_  r3   r/  rc  rd  r  r\  rX  rZ  rR  ra  r.   rÑ   r]  rY  r[  )NNN)NNNNNNN)
r]   r^   r_   r`   rU   r‡  rÉ   ra   rb   rc   rd   s   @@r:   rJ  rJ  u  s#   ù‡ € ÷1:õ 1:òfn÷`Gô Gr<   rJ  c                   ó   € ] tR tRtRtR# )r^  ic  r\   Nrw   r\   r<   r:   r^  r^  c  rx   r<   r^  c                   ó   € ] tR tRtRtR# )ÚEvollaRotaryEmbeddingig  r\   Nrw   r\   r<   r:   rœ  rœ  g  rx   r<   rœ  c                   ó   € ] tR tRtRtR# )Ú	EvollaMLPik  r\   Nrw   r\   r<   r:   rž  rž  k  rx   r<   rž  c                   ó   € ] tR tRtRtR# )ÚEvollaAttentionio  r\   Nrw   r\   r<   r:   r   r   o  rx   r<   r   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚEvollaDecoderLayeris  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r,   rY   Ú	layer_idx)r)   r  )r9   re   s   "€r:   r;   ÚEvollaDecoderLayer.__annotate__t  s   ø€ ÷ ñ ™|ð ¹ñ r<   c                óÖ   <€ \         SV `  W4       V^,           \        VP                  VP                  ,          ^4      ,          ^ 8X  d   \        VVP                  R7      V n        R# R# )é   )rL  N)rT   rU   ÚmaxÚnum_hidden_layersÚaligner_num_add_layersrJ  r  Úadapter©rX   rY   r¤  rZ   s   &&&€r:   rU   ÚEvollaDecoderLayer.__init__t  sY   ø€ Ü‰Ñ˜Ô+Ø˜�MœS ×!9Ñ!9¸V×=ZÑ=ZÕ!ZÐ\]Ó^Õ^ÐbcÔcÜ>ØØ$*×$6Ñ$6ôˆDŽLñ dr<   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          /# )r,   r‰   r¿   Nr1   rV   r”  Ú	use_cacherŽ  r�  r�  r‘  r’  r“  rr  )r6   r7   rq   rC  r   rì   )r9   re   s   "€r:   r;   r¥  |  s  ø€ ÷ 3ñ 3á—|‘|ð3ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð3ñ Ÿ™ tÕ+ð	3ñ
 ×&Ñ&¨Õ-ð3ñ  �ð3ñ ˜$•;ð3ñ !Ÿ<™<¨$Õ.ð3ñ #Ÿ\™\¨DÕ0ð3ñ —|‘| dÕ*ð3ñ "ŸL™L¨4Õ/ð3ñ $Ÿl™l¨TÕ1ð3ñ Ÿ™ tÕ+ð3ñ Ÿ™¨Õ,ñ3r<   c                ó,  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  ppWñ,           pTpV P                  V4      pV P                  V4      pWñ,           p\	        V R4      '       d   V P                  VVVV	VV
VVR7      pV# )	r‰   r1   rV   r”  r¯  r¿   r«  )rn  rŽ  r�  r�  rr  r‘  r’  r“  r\   )Úinput_layernormÚ	self_attnÚpost_attention_layernormÚmlpÚhasattrr«  )rX   r‰   r¿   r1   rV   r”  r¯  rŽ  r�  r�  r‘  r’  r“  rr  r4   r˜  r—   s   &&&&&&&&&&&&&&,  r:   rÉ   ÚEvollaDecoderLayer.forward|  sç   € ð" !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆ�qð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆä�4˜×#Ò#Ø ŸL™LØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ó 	ˆMð Ðr<   )r«  )NNNNFNNNNNNNr  rd   s   @@r:   r¢  r¢  s  s   ù‡ € ÷ó ÷3÷ 3ò 3r<   r¢  c                   ób   a € ] tR tRt o RtRtRt. ROt]P                  ! 4       R 4       t
RtV tR# )ÚEvollaPreTrainedModeli²  Fc                óØ  € V P                   P                  p\        P                  ! W4       \	        V\
        4      '       dm   \        P                  ! VP                  4       \        P                  ! VP                  4       \        P                  ! VP                  P                  4       R# \	        V\        4      '       d&   \        P                  ! VP                  R VR7       R# R# )rQ  )ÚmeanÚstdN)rY   Úinitializer_ranger   r’   r“   rJ  r”   Úzeros_rc  rd  Úones_r_  Úweightr  Únormal_rï   )rX   r-   r»  s   && r:   r’   Ú#EvollaPreTrainedModel._init_weights½  s�   € à�k‰k×+Ñ+ˆÜ×%Ò% dÔ3Ü�fÔA×BÒBÜ�KŠK˜×-Ñ-Ô.Ü�KŠK˜Ÿ™Ô(Ü�JŠJ�v×,Ñ,×3Ñ3Ö4Ü˜Ô A×BÒBÜ�LŠL˜Ÿ™¨c°s×;ñ Cr<   r\   N)r¢  r‚   r  rJ  )r]   r^   r_   r`   rœ   rž   rŸ   r›   r6   r¡   r’   ra   rb   r9  s   @r:   r¸  r¸  ²  s9   ø‡ € Ø ÐØÐØ"'ÐòÐð ‡]‚]ƒ_ñ<ó ö<r<   r¸  c                   ó~   a a€ ] tR tRt oV3R lV 3R lltR tR t]]]	R	V3R lR ll4       4       4       t
RtVtV ;t# )
ÚEvollaModeliÉ  c                ó    <€ V ^8„  d   QhRS[ /# ri   r  )r9   re   s   "€r:   r;   ÚEvollaModel.__annotate__Ê  s   ø€ ÷ ñ ™|ñ r<   c           
     óP  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! V P                  VP                  V P                  4      V n        \        VR 7      V n
        \
        P                  ! \        VP                  4       Uu. uF  p\        VVR7      NK  	  up4      V n        \!        VP                  VP"                  R7      V n        \'        VRR4      V n        \+        VR 7      V n        V P/                  4        R# u upi )r©   )rY   r¤  )ÚepsÚgradient_checkpointingFN)rT   rU   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr  Úembed_tokensr;  Úprotein_encoderr  r!  r©  r¢  r   r^  Úrms_norm_epsr  rF   rÈ  rœ  Ú
rotary_embr­   r¬  s   && €r:   rU   ÚEvollaModel.__init__Ê  sð   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨¯©¸&×:LÑ:LÈd×N^ÑN^Ó_ˆÔÜ3¸6ÔBˆÔÜ—m’mô "' v×'?Ñ'?Ô!@óñ
 "A�Iô	 #Ø!Ø'÷ñ "Añó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü&-¨fÐ6NÐPUÓ&VˆÔ#Ü/°vÔ>ˆŒØ�‰Öùòs   Â&D#c                ó   € V P                   # rS   ©rÍ  r²   s   &r:   r³   Ú EvollaModel.get_input_embeddingsß  s   € Ø× Ñ Ð r<   c                ó   € Wn         R # rS   rÓ  r¶   s   &&r:   r·   Ú EvollaModel.set_input_embeddingsâ  s   € Ø!Ör<   c                óè  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          /# )r,   rº   Nr1   rV   r”  r¾   r¯  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsr’  r“  rp   )r6   rC  r7   r   r6  rì   rq   r   )r9   re   s   "€r:   r;   rÅ  è  s  ø€ ÷ ]ñ ]á×#Ñ# dÕ*ð]ñ Ÿ™ tÕ+ð]ñ ×&Ñ&¨Õ-ð	]ñ
  �ð]ñ ×(Ñ(¨4Õ/ð]ñ ˜$•;ð]ñ !×+Ñ+¨dÕ2ð]ñ !&§¡¨tÕ 3ð]ñ ×*Ñ*¨TÕ1ð]ñ ×$Ñ$ tÕ+ð]ñ $Ÿl™l¨TÕ1ð]ñ Ÿ™ tÕ+ð]ñ 
Ñ(Õ	(ñ]r<   c                óT  € VRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pRpRpVef   Veb   V P                  VVR7      pVP                  p\
        P                  ! VP                  ^ ,          VP                  \
        P                  R7      p\        V P                  VVVR7      pTpV P                  VVR7      pV P                    F$  pV! V3R	VR
VRVRVRVRV	RV
RVRVRVRVRV/VB pK&  	  V P#                  V4      p\%        VVR7      pV# )aÑ  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
structure_feats (torch.FloatTensor):
    The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
msa_feats (torch.FloatTensor):
    The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
structure_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummy input for now.
msa_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummy input for now.
Nz:You must specify exactly one of input_ids or inputs_embedsr©   r¼   r½   )rm   r+  )rY   r¾   r1   r”  )rV   r1   rV   r”  r¯  rŽ  r�  r�  r‘  r’  r“  rr  r¿   )rÀ   r”  )rj  rÍ  r   rY   Úget_seq_lengthr6   rÂ   ré   rm   rÃ   rÎ  r5  rÁ   rì   r
   rÐ  r   r  r   )rX   rº   r1   rV   r”  r¾   r¯  rØ  rÙ  rÚ  rÛ  r’  r“  r4   Úpast_seen_tokensÚprotein_featsr‘  Úprotein_outputsÚcausal_maskr‰   r¿   Údecoder_layerÚoutputs   &&&&&&&&&&&&&,         r:   rÉ   ÚEvollaModel.forwardå  s
  € ðB ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLàˆØ!ÐàÒ(Ð-CÒ-OØ"×2Ñ2Ø+Ø5ð 3ó ˆOð ,×FÑFˆMÜ!&§¢Ø!×'Ñ'¨Õ*Ø(×/Ñ/Ü—j‘jô"Ðô )Ø—;‘;Ø'Ø)Ø+ô	
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð #0ðð %4ðð (ðð $6ðð &:ðð  .ðð !/ðð %8ðð ñŠMñ )ð$ Ÿ	™	 -Ó0ˆä(Ø+Ø+ô
ˆð ˆr<   )rÍ  rÈ  r   r  rÊ  rÎ  rÐ  rË  )NNNNNNNNNNNN)r]   r^   r_   r`   rU   r³   r·   r   r   r   rÉ   ra   rb   rc   rd   s   @@r:   rÃ  rÃ  É  sK   ù‡ € ÷ó ò*!ò"ð ØØ÷]ñ ]ó ó  ó ÷]ð ]r<   rÃ  c                   óh   a a€ ] tR tRt oV 3R ltR tR t]]RV3R lR ll4       4       t	Rt
VtV ;t# )	ÚEvollaForProteinText2TextiH  c                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  V P                  R R7      V n        V P                  4        R# r  )
rT   rU   rÃ  r?  rË  r   rÖ   r  Úlm_headr­   rW   s   &&€r:   rU   Ú"EvollaForProteinText2Text.__init__I  sQ   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°T·_±_È5ÔQˆŒà�‰Ör<   c                ó6   € V P                   P                  4       # rS   )r?  r³   r²   s   &r:   r³   Ú.EvollaForProteinText2Text.get_input_embeddingsQ  s   € Ø�z‰z×.Ñ.Ó0Ð0r<   c                ó8   € V P                   P                  V4      # rS   )r?  r·   r¶   s   &&r:   r·   Ú.EvollaForProteinText2Text.set_input_embeddingsT  s   € Ø�z‰z×.Ñ.¨uÓ5Ð5r<   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          /# )
r,   rº   Nr1   r¾   ÚlabelsrØ  rÙ  r¯  Úlogits_to_keep)r6   rC  r7   r6  rì   r  )r9   re   s   "€r:   r;   Ú&EvollaForProteinText2Text.__annotate__Y  s°   ø€ ÷ Bñ Bá×#Ñ# dÕ*ðBñ Ÿ™ tÕ+ðBñ ×(Ñ(¨4Õ/ð	Bñ
 × Ñ  4Õ'ðBñ !×+Ñ+¨dÕ2ðBñ !&§¡¨tÕ 3ðBñ ˜$•;ðBñ ™eŸl™lÕ*ñBr<   c	                ó†  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe#   V P                  ! RR	VR
VRV P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      pV# )a|  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

Example:

```python
>>> from transformers import EvollaProcessor, EvollaForProteinText2Text
>>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
>>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

>>> protein_information = {
    "aa_seq": "your amino acid sequence",
    "foldseek": "your foldseek sequence",
}
>>> question = "What is the function of this protein?"
>>> message = [
    {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
    {"role": "user", "content": question},
]

>>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
>>> outputs = model.generate(**inputs)

>>> print(processor.batch_decode(outputs, skip_special_tokens=True))
```rº   r1   r¾   rØ  rÙ  r¯  Nrâ   Úlogitsrï  rË  )Úlossró  r”  r‰   rŠ   r\   )r?  rÀ   r“   r  Úslicerè  Úloss_functionrË  r   r”  r‰   rŠ   )rX   rº   r1   r¾   rï  rØ  rÙ  r¯  rð  r4   Úoutputsr‰   Úslice_indicesró  rô  Ú
lm_outputss   &&&&&&&&&,      r:   rÉ   Ú!EvollaForProteinText2Text.forwardW  sÿ   € ðT ,0¯:ª:ñ ,
Øð,
à)ð,
ð (ð,
ð 0ð	,
ð
 $:ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñi¨VÐi¸FÐiÈtÏÉÐiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ˆ
ð Ðr<   )rè  r?  rË  )NNNNNNNr   )r]   r^   r_   r`   rU   r³   r·   r   r   rÉ   ra   rb   rc   rd   s   @@r:   ræ  ræ  H  s>   ù‡ € õò1ò6ð Ø÷Bñ Bó ó ÷Bð Br<   ræ  )NrQ  )ræ  rÃ  r¸  )XÚdataclassesr   r6   r   Ú r   r”   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr	   r
   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úesm.modeling_esmr   r   r   r   r   r   r   r   r    r!   Úllama.modeling_llamar"   r#   r$   r%   r&   r'   r(   Úconfiguration_evollar)   r*   Ú
get_loggerr]   ÚloggerrO   rQ   rg   rv   rz   r|   r~   r€   r‚   r„   r†   rˆ   r¤   r5   rÌ   r  r  r3  r;  rJ  r^  rœ  rž  r   r¢  r¸  rÃ  ræ  Ú__all__r\   r<   r:   Ú<module>r     s  ðõ "ã Ý å &ß .Ý )ß J÷ó õ .Ý &÷ó õ 8ß E÷÷ ÷ ÷÷ ñ ÷ =ð 
×	Ò	˜HÓ	%€÷ %ôF˜]ô ô?Ð"4ô ?ô	Ð 0ô 	ô	˜]ô 	ô	˜Lô 	ô	˜ô 	ô	˜ô 	ô	˜ô 	ô	˜*ô 	ô	˜ô 	ð ôC /ó Có ðCô01
Ð!<ô 1
ôh7 ¨¯	©	ô 7 ôtA˜Ÿ	™	ô Aô'.¨¯	©	ô '.ðT Ø
ô
< kó 
<ó ó ð
<ô
˜2Ÿ9™9ô 
ô$k¨"¯)©)ô kô\	�Lô 	ô	Ð0ô 	ô	�ô 	ô	�nô 	ô<Ð*ô <ô~<Ð0ô <ô.|Ð'ô |ô~SÐ 5°ô Sòl P‚r<   