+
    œ°³jÜú  ã                   ó˜  € ^ RI t ^ RIHt ^ RIHt ^ RIt^ RIHt ^RIHt	 ^RI
Ht ^RIHtHt ^RIHt ^R	IHtHt ^R
IHtHt ^RIHt ^RIHtHtHtHtHt ^RIH t H!t! ^RI"H#t#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t* ^RI+H,t, ^RI-H.t.H/t/ ^RI0H1t1H2t2 ^RI3H4t4H5t5 R t6 ! R R]Pn                  4      t8 ! R R]Pn                  4      t9R t:]! R4      RSR l4       t;RTR R llt<]! ];4       ! R R ]Pn                  4      4       t= ! R! R"]Pn                  4      t> ! R# R$]Pn                  4      t?R% t@ ! R& R']Pn                  4      tA ! R( R)]Pn                  4      tB ! R* R+]4      tC ! R, R-]Pn                  4      tD ! R. R/]Pn                  4      tE]) ! R0 R1]$4      4       tF ! R2 R3]F4      tG ! R4 R5]Pn                  4      tH ! R6 R7]Pn                  4      tI ! R8 R9]Pn                  4      tJ])] ! R: R;]4      4       4       tK ! R< R=]Pn                  4      tL ! R> R?]Pn                  4      tM]! R@4       ! RA RB]Pn                  4      4       tN ! RC RD]Pn                  4      tO ! RE RF]Pn                  4      tPRG RH ltQ]! ];4       ! RI RJ]Pn                  4      4       tR ! RK RL]4      tS]) ! RM RN]$4      4       tT ! RO RP]T4      tU ! RQ RR]T]4      tV. RUOtWR# )Vé    N)ÚCallable)Ú	dataclass)Únn)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)ÚEvollaConfigÚSaProtConfigc                óÊ   € V P                  V4      P                  4       p\        P                  ! V^R7      P	                  V4      V,          pVP                  4       V,           # )zí
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's `utils.make_positions`.

Args:
    x: torch.Tensor x:

Returns: torch.Tensor
©Údim)ÚneÚintÚtorchÚcumsumÚtype_asÚlong)Ú	input_idsÚpadding_idxÚmaskÚincremental_indicess   &&  Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/evolla/modeling_evolla.pyÚ"create_position_ids_from_input_idsr2   4   sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜŸ,š, t°Ô3×;Ñ;¸DÓAÀDÕHÐØ×#Ñ#Ó%¨Õ3Ð3ó    c                   óF   a a€ ] tR t^Dt oRtV 3R ltRR ltR tRtVt	V ;t
# )ÚEvollaSaProtEmbeddingszN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                ó   <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        VP                  '       d3   \        P                  ! VP
                  VP                  R7      V n        MRV n        \        P                  ! VP                  4      V n        \        VRR4      V n        VP                  V n        V P                   R8X  d=   \        P                  ! VP$                  VP
                  V P"                  R 7      V n        VP(                  V n        VP*                  V n        R# ))r.   ©ÚepsNÚposition_embedding_typeÚabsolute)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutÚgetattrr9   r.   Úmax_position_embeddingsÚposition_embeddingsÚtoken_dropoutÚmask_token_id©ÚselfÚconfigÚ	__class__s   &&€r1   r<   ÚEvollaSaProtEmbeddings.__init__I   sõ   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔà×'×'Ð'Ü Ÿlšl¨6×+=Ñ+=À6×CXÑCXÔYˆD�Oà"ˆDŒOÜ—z’z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$à!×.Ñ.ˆÔØ×'Ñ'¨:Ô5Ü')§|¢|Ø×.Ñ.°×0BÑ0BÐPT×P`ÑP`ô(ˆDÔ$ð $×1Ñ1ˆÔØ#×1Ñ1ˆÖr3   c                óF  € Vf,   Ve   \        WP                  4      pMV P                  V4      pVf   V P                  V4      pTpV P                  '       dÌ   VeÈ   VP                  WP                  8H  P                  R4      R4      pRpVe   VP                  R4      MVP                  ^,          pWP                  8H  P                  R4      P                  4       V,          pV^V,
          ,          ^V,
          R,          ,          P                  VP                  4      pV P                  R8X  d   V P                  V4      p	WY,           pV P                  e   V P                  V4      pVe1   WRP                  R4      ,          P                  VP                  4      pV# )Nç        r:   éÿÿÿÿg¸…ëQ¸¾?)ºNNNNN)r2   r.   Ú&create_position_ids_from_inputs_embedsrA   rL   Úmasked_fillrM   Ú	unsqueezeÚsumÚshapeÚfloatÚtoÚdtyper9   rK   rE   )
rO   r-   Úattention_maskÚposition_idsÚinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedrK   s
   &&&&&     r1   ÚforwardÚEvollaSaProtEmbeddings.forward]   sƒ  € ð ÒØÒ$äAÀ)×M]ÑM]Ó^‘à#×JÑJÈ=ÓY�àÒ Ø ×0Ñ0°Ó;ˆMð #ˆ
ð ××Ð )Ò"7Ø#×/Ñ/°×>PÑ>PÑ1P×0[Ñ0[Ð\^Ó0_ÐadÓeˆJØ)ÐØ4BÒ4N˜.×,Ñ,¨RÔ0ÐT]×TcÑTcÐdeÕTfˆKØ#,×0BÑ0BÑ#B×"GÑ"GÈÓ"K×"QÑ"QÓ"SÐVaÕ"aÐØ$¨Ð,<Õ(<Õ=ÀÐEXÕAXÐZgÕ@hÕh×lÑlØ× Ñ óˆJð ×'Ñ'¨:Ô5Ø"&×":Ñ":¸<Ó"HÐØ#Õ9ˆJà�?‰?Ò&ØŸ™¨Ó4ˆJØÒ%Ø$×'?Ñ'?ÀÓ'CÕC×GÑGÈ
×HXÑHXÓYˆJð Ðr3   c                ó.  € VP                  4       RR pV^,          p\        P                  ! V P                  ^,           W0P                  ,           ^,           \        P                  VP
                  R7      pVP                  ^ 4      P                  V4      # )z¯
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
N©r^   ÚdevicerU   )Úsizer)   Úaranger.   r,   rj   rY   Úexpand)rO   ra   Úinput_shapeÚsequence_lengthr`   s   &&   r1   rW   Ú=EvollaSaProtEmbeddings.create_position_ids_from_inputs_embedsŽ   s~   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a�.ˆä—|’|Ø×Ñ˜qÕ  /×4DÑ4DÕ"DÀqÕ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r3   )rH   rE   rM   r.   r9   rK   rL   rA   ©NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r<   rf   rW   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rQ   Ú__classdict__s   @@r1   r5   r5   D   s   ù‡ € ñõ2ô(/÷b=ò =r3   r5   c                   óÖ   a a€ ] tR t^ t oRt]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]	P                  ! 4       ]RR	 l4       4       tR
tVtV ;t# )ÚEvollaSaProtRotaryEmbeddingzî
Rotary position embeddings.
Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
rj   ú5.18©Úversionc                ó    <€ V ^8„  d   QhRS[ /# ©é   rP   ©r#   )Úformatr{   s   "€r1   Ú__annotate__Ú(EvollaSaProtRotaryEmbedding.__annotate__§   s   ø€ ÷ Cñ C™|ñ Cr3   c                óÈ   <€ \         SV `  4        Wn        / V n        V P	                  V P                  4      w  r4\
        P                  ! V4      V n        \        V R V4       R# )Úattention_scalingN)	r;   r<   rP   Ú	rope_typeÚcompute_default_rope_parametersr   ÚBufferÚinv_freqÚsetattr)rO   rP   rj   Úcurr_inv_freqÚcurr_attention_scalingrQ   s   &&&  €r1   r<   Ú$EvollaSaProtRotaryEmbedding.__init__¦   sR   ø€ ä‰ÑÔàŒØˆŒà04×0TÑ0TÐUY×U`ÑU`Ó0aÑ-ˆÜŸ	š	 -Ó0ˆŒÜ�Ð)Ð+AÖBr3   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# ©rƒ   rP   Úreturn)r#   Útupler)   ÚTensorr\   )r…   r{   s   "€r1   r†   r‡   ³   s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r3   c           	     ó"  € V P                   p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.

Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Úhead_dimNç      ð?©r^   )Ú
rope_thetarI   r?   Únum_attention_headsr)   rl   r\   r]   ©rP   rj   ÚkwargsÚbaser&   Úattention_factorr�   s   &&,    r1   r‹   Ú;EvollaSaProtRotaryEmbedding.compute_default_rope_parameters±   sy   € ð × Ñ ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                óˆ  € \        V R 4      p\        V R4      pVR	,          P                  4       P                  VP                  ^ ,          R
^4      P	                  VP
                  4      pVR,          P                  4       p\        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p	\        P                  ! W™3R
R7      p
V
P                  4       V,          pV
P                  4       V,          pRRR4       XP	                  VP                  R7      XP	                  VP                  R7      3#   + '       g   i     LH; i)r�   r‰   NÚmpsÚcpuF©Údevice_typeÚenabledr%   rš   ©NrV   NrU   ©rV   NrV   )rI   r\   rm   r[   r]   rj   Ú
isinstanceÚtypeÚstrr   Ú	transposer)   ÚcatÚcosÚsinr^   )rO   Úxr`   Ú
layer_typer�   r‰   Úinv_freq_expandedÚposition_ids_expandedr¦   ÚfreqsÚembr¯   r°   s   &&&&         r1   rf   Ú#EvollaSaProtRotaryEmbedding.forwardÆ   sQ  € ô ˜4 Ó,ˆÜ# DÐ*=Ó>Ðà$ ]Õ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÕCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)Ð/Õ/ˆCØ—'‘'“)Ð/Õ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã4A=F1Æ1G	)rP   r�   rŠ   ©N)rr   rs   rt   ru   rv   r   r<   Ústaticmethodr‹   r)   Úno_gradr   rf   rw   rx   ry   rz   s   @@r1   r}   r}       s|   ù‡ € ññ
 �X vÔ.÷Cô Có /ðCð Ù�X vÔ.÷5ñ 5ó /ó ð5ð& ‡]‚]ƒ_Øó<ó ó ÷<ð <r3   r}   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # )z*Rotates half the hidden dims of the input..Nr%   rU   )r[   r)   r®   )r±   Úx1Úx2s   &  r1   Úrotate_halfr¾   Ù   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r3   Úrotary_pos_embc                ó’  € V P                   pVP                  V4      pVP                  V4      pV P                  4       V,          \        V P                  4       4      V,          ,           pVP                  4       V,          \        VP                  4       4      V,          ,           pVP	                  V4      VP	                  V4      3# )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)r^   rY   r\   r¾   r]   )ÚqÚkr¯   r°   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds   &&&&&   r1   Úapply_rotary_pos_embrÇ   à   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�w‰w‹y˜3�¤;¨q¯w©w«yÓ#9¸CÕ#?Õ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr3   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
rƒ   ÚmoduleÚqueryÚkeyÚvaluer_   NÚscalingrH   rž   )r   ÚModuler)   r–   r\   r   r   )r…   s   "r1   r†   r†   û   s{   € ÷  %ñ  %Ü�I‰Ið %ä�<‰<ð %ô 
�‰ð %ô �<‰<ð	 %ô
 —L‘L 4Õ'ð %ô �T�\ð %ô ð %ô Ô'Õ(ñ %r3   c                 óü  € Vf   VP                  R4      R,          p\        V R^4      p\        W(4      p\        W84      p\        P                  ! WP                  ^^4      4      V,          p	Ve	   W”,           p	\        P                  P                  V	RR7      p	\        P                  P                  W–V P                  R7      p	\        P                  ! W“4      p
V
P                  ^^4      P                  4       p
W©3# )NÚnum_key_value_groupsr%   )ÚpÚtrainingrU   ç      à¿)rk   rI   Ú	repeat_kvr)   Úmatmulr­   r   Ú
functionalÚsoftmaxrH   rÒ   Ú
contiguous)rÉ   rÊ   rË   rÌ   r_   rÍ   rH   rž   Ún_repÚattn_weightsÚattn_outputs   &&&&&&&,   r1   Úeager_attention_forwardrÜ   û   sÖ   € ð ‚Ø—*‘*˜R“. DÕ(ˆô
 �FÐ2°AÓ6€EÜ
�CÓ
€CÜ�eÓ#€Eô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r3   c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚEvollaSaProtSelfAttentioni  c                ó–  <€ \         SV `  4        Wn        VP                  VP                  ,          ^ 8w  d9   \        VR4      '       g'   \        RVP                   RVP                   R24      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n	        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        VP                  V n        T;'       g    \#        VRR4      V n        RV n        VP(                  V n        W0n        V P(                  ;'       d    V'       * V n        R# )	r   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)r9   r:   r™   N)r;   r<   rP   r?   rœ   ÚhasattrÚ
ValueErrorr(   Úattention_head_sizeÚall_head_sizer   ÚLinearrÊ   rË   rÌ   Úattention_probs_dropout_probrH   rI   r9   rÍ   Ú
is_decoderÚ	layer_idxÚ	is_causal)rO   rP   r9   ré   Úis_cross_attentionrQ   s   &&&&&€r1   r<   Ú"EvollaSaProtSelfAttention.__init__   so  ø€ Ü‰ÑÔØŒà×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
à×:Ñ:ˆŒà'>÷ (
ð (
Ä'ØÐ-¨zóC
ˆÔ$ð ˆŒØ ×+Ñ+ˆŒØ"ŒØŸ™×CÐCÐ1CÔ-CˆŽr3   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[ P                  ,          /# )	rƒ   Úhidden_statesr_   NÚencoder_hidden_statesÚencoder_attention_maskrK   rž   r”   )r)   r–   ÚFloatTensorr   r   r•   )r…   r{   s   "€r1   r†   Ú&EvollaSaProtSelfAttention.__annotate__<  sŽ   ø€ ÷ .)ñ .)á—|‘|ð.)ñ ×)Ñ)¨DÕ0ð.)ñ  %×0Ñ0°4Õ7ð	.)ñ
 !&× 1Ñ 1°DÕ 8ð.)ñ #Ÿ\™\¨DÕ0ð.)ñ Ñ+Õ,ð.)ñ 
‰u�|‰|Õ	ñ.)r3   c                ó>  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	VR Jp
V
'       d   TMTpV
'       d   TMTpV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      pW�P                  R,          ,          p	V P                  R8X  d   Vw  rï\        WœWï^R7      w  rœ\        P                  ! V P                  P                  \        4      pT! V V	VVV3RV P                  '       g   RMV P                  RV P                   /VB w  ppVP"                  ! . VORN5!  P%                  4       pVV3# )NÚrotary)rÃ   rH   rT   rÍ   rU   rÓ   )r[   rä   rÊ   Úviewr­   rË   rÌ   r9   rÇ   r   Úget_interfacerP   Ú_attn_implementationrÜ   rÒ   rH   rÍ   ÚreshaperØ   )rO   rî   r_   rï   rð   rK   rž   rn   Úhidden_shapeÚquery_layerrë   Úcurrent_statesÚ	key_layerÚvalue_layerr¯   r°   Úattention_interfacerÛ   rÚ   s   &&&&&&,            r1   rf   Ú!EvollaSaProtSelfAttention.forward<  sŸ  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆà—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆà2¸$Ð>Ðß2DÑ.È-ˆß3EÑ/È>ˆØ—H‘H˜^Ó,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó0×5Ñ5°lÓC×MÑMÈaÐQRÓSˆð "×$<Ñ$<¸dÕ$BÕBˆà×'Ñ'¨8Ô3Ø*‰HˆCÜ%9¸+ÐRUÐjkÔ%lÑ"ˆKä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ˜LÐ(Ð(r3   )rå   rä   rP   rH   rê   rè   rË   ré   rœ   r9   rÊ   rÍ   rÌ   )NNFrq   ©	rr   rs   rt   ru   r<   rf   rw   rx   ry   rz   s   @@r1   rÞ   rÞ     s   ù‡ € ÷D÷8.)÷ .)ò .)r3   rÞ   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚEvollaSaProtSelfOutputim  c                óÎ   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        R # r¸   )	r;   r<   r   ræ   r?   ÚdenserF   rG   rH   rN   s   &&€r1   r<   ÚEvollaSaProtSelfOutput.__init__n  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŽr3   c                óZ   € V P                  V4      pV P                  V4      pW,           pV# r¸   ©r  rH   ©rO   rî   Úinput_tensors   &&&r1   rf   ÚEvollaSaProtSelfOutput.forwards  ó,   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%Õ4ˆØÐr3   r  r   rz   s   @@r1   r  r  m  ó   ù‡ € õ>÷
ò r3   r  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚEvollaSaProtAttentioniz  c                óÊ   <€ \         SV `  4        \        WVR 7      V n        \	        V4      V n        \        P                  ! VP                  VP                  R7      V n        R# ))ré   rë   r7   N)
r;   r<   rÞ   rO   r  Úoutputr   rC   r?   rD   )rO   rP   ré   rë   rQ   s   &&&&€r1   r<   ÚEvollaSaProtAttention.__init__{  sG   ø€ Ü‰ÑÔÜ-¨fÐ^pÔqˆŒ	Ü,¨VÓ4ˆŒäŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr3   c                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# ©rƒ   rK   Nrž   ©r)   r–   r   r   )r…   r{   s   "€r1   r†   Ú"EvollaSaProtAttention.__annotate__‚  s0   ø€ ÷ ñ ñ #Ÿ\™\¨DÕ0ðñ Ñ+Õ,ñr3   c                ó„   € V P                  V4      pV P                  ! V3R VRVRVRV/VB w  r‰V P                  W�4      pV# )r_   rï   rð   rK   )rC   rO   r  )
rO   rî   r_   rï   rð   rK   rž   Úhidden_states_lnrÛ   Ú_s
   &&&&&&,   r1   rf   ÚEvollaSaProtAttention.forward‚  sl   € ð  Ÿ>™>¨-Ó8ÐØŸšØñ
à)ð
ð #8ð
ð $:ð	
ð
 !4ð
ð ñ
‰ˆð —k‘k +Ó=ˆØÐr3   )rC   r  rO   )NFrq   r   rz   s   @@r1   r  r  z  s   ù‡ € ÷U÷÷ ò r3   r  c                óŽ   € V R,          R\         P                  ! V \        P                  ! R4      ,          4      ,           ,          # )zr
This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results.
ç      à?r™   g       @)r)   ÚerfÚmathÚsqrt)r±   s   &r1   Úgelur  ˜  s.   € ð ˆs�7�cœEŸIšI a¬$¯)ª)°C«.Õ&8Ó9Õ9Õ:Ð:r3   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚEvollaSaProtIntermediateiŸ  c                ó„   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        R # r¸   )r;   r<   r   ræ   r?   Úintermediate_sizer  rN   s   &&€r1   r<   Ú!EvollaSaProtIntermediate.__init__   s,   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŽ
r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rƒ   rî   r”   ©r)   r–   )r…   r{   s   "€r1   r†   Ú%EvollaSaProtIntermediate.__annotate__¤  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ó>   € V P                  V4      p\        V4      pV# r¸   )r  r  )rO   rî   s   &&r1   rf   Ú EvollaSaProtIntermediate.forward¤  s    € ØŸ
™
 =Ó1ˆÜ˜]Ó+ˆØÐr3   )r  r   rz   s   @@r1   r!  r!  Ÿ  s   ù‡ € õM÷÷ ð r3   r!  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚEvollaSaProtOutputiª  c                óÎ   <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP                  4      V n	        R # r¸   )
r;   r<   r   ræ   r#  r?   r  rF   rG   rH   rN   s   &&€r1   r<   ÚEvollaSaProtOutput.__init__«  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŽr3   c                óZ   € V P                  V4      pV P                  V4      pW,           pV# r¸   r  r  s   &&&r1   rf   ÚEvollaSaProtOutput.forward°  r  r3   r  r   rz   s   @@r1   r,  r,  ª  r  r3   r,  c                   óN   a a€ ] tR tRt oV 3R ltRV3R lR lltR tRtVtV ;t	# )ÚEvollaSaProtLayeri·  c                óâ  <€ \         SV `  4        VP                  V n        ^V n        \	        V4      V n        VP                  V n        VP                  V n        V P                  '       d3   V P                  '       g   \        V  R24      h\	        VRR7      V n	        \        V4      V n        \        V4      V n        \        P                  ! VP                   VP"                  R7      V n        R# )é   z> should be used as a decoder model if cross attention is addedT)rë   r7   N)r;   r<   Úchunk_size_feed_forwardÚseq_len_dimr  Ú	attentionrè   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr!  Úintermediater,  r  r   rC   r?   rD   rN   s   &&€r1   r<   ÚEvollaSaProtLayer.__init__¸  sº   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ.¨vÓ6ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#×#Ð#Ø—?—?�?Ü" d VÐ+iÐ#jÓkÐkÜ"7¸ÐSWÔ"XˆDÔÜ4°VÓ<ˆÔÜ(¨Ó0ˆŒÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr3   c                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# r  r  )r…   r{   s   "€r1   r†   ÚEvollaSaProtLayer.__annotate__Ç  s0   ø€ ÷ !ñ !ñ #Ÿ\™\¨DÕ0ð!ñ Ñ+Õ,ñ!r3   c                óú   € V P                   ! V3R VRV/VB pV P                  '       dA   Ve=   \        V R4      '       g   \        RV  R24      hV P                  ! V3R VRVRVRV/VB pV P                  V4      pV# )r_   rK   r:  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rï   rð   )r7  rè   râ   ÚAttributeErrorr:  Úfeed_forward_chunk)	rO   rî   r_   rï   rð   rK   rž   Úattention_outputÚlayer_outputs	   &&&&&&,  r1   rf   ÚEvollaSaProtLayer.forwardÇ  sÑ   € ð  Ÿ>š>Øñ
à)ð
ð !4ð
ð ñ	
Ðð �?�?ˆ?Ð4Ò@Ü˜4Ð!1×2Ò2Ü$Ø=¸d¸Vð D`ð `óð ð
  $×2Ò2Ø ñ à-ð ð '<ð ð (>ð	 ð
 %8ð ð ñ Ðð ×.Ñ.Ð/?Ó@ˆØÐr3   c                ól   € V P                  V4      pV P                  V4      pV P                  W14      pV# r¸   )rC   r;  r  )rO   rB  Úattention_output_lnÚintermediate_outputrC  s   &&   r1   rA  Ú$EvollaSaProtLayer.feed_forward_chunkê  s9   € Ø"Ÿn™nÐ-=Ó>ÐØ"×/Ñ/Ð0CÓDÐØ—{‘{Ð#6ÓIˆØÐr3   )	rC   r8  r7  r5  r:  r;  rè   r  r6  rq   )
rr   rs   rt   ru   r<   rf   rA  rw   rx   ry   rz   s   @@r1   r2  r2  ·  s    ù‡ € õU÷!ò !÷Fò r3   r2  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚEvollaSaProtEncoderiñ  c                ó8  <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        \        P                  ! VP                  VP                  R 7      V n        RV n        R# u upi )r7   FN)r;   r<   rP   r   Ú
ModuleListÚrangeÚnum_hidden_layersr2  ÚlayerrC   r?   rD   Úemb_layer_norm_afterÚgradient_checkpointing)rO   rP   r  rQ   s   && €r1   r<   ÚEvollaSaProtEncoder.__init__ò  sr   ø€ Ü‰ÑÔØŒÜ—]’]ÄuÈV×MeÑMeÔGfÓ#gÑGfÀ!Ô$5°fÖ$=ÑGfÑ#gÓhˆŒ
Ü$&§L¢L°×1CÑ1CÈ×I^ÑI^Ô$_ˆÔ!Ø&+ˆÖ#ùò $hs   ½Bc                óX   <€ V ^8„  d   QhRS[ P                  R,          RS[S[,          /# r  r  )r…   r{   s   "€r1   r†   Ú EvollaSaProtEncoder.__annotate__ú  s4   ø€ ÷ Sñ Sñ #Ÿ\™\¨DÕ0ðSñ Ñ+Õ,ñSr3   c                ó¾   € \        V P                  4       F  w  rxV! V3R VRVRVRV/VB pK  	  V P                  '       d   V P                  V4      p\        VR7      # )r_   rï   rð   rK   )Úlast_hidden_state)Ú	enumeraterO  rP  r   )	rO   rî   r_   rï   rð   rK   rž   ÚiÚlayer_modules	   &&&&&&,  r1   rf   ÚEvollaSaProtEncoder.forwardù  s   € ô  )¨¯©Ö4‰OˆAÙ(Øñà-ðð '<ðð (>ð	ð
 %8ðð ñŠMñ  5ð ×$×$Ð$Ø ×5Ñ5°mÓDˆMä1ÀMÔRÐRr3   )rP   rP  rQ  rO  rq   ©
rr   rs   rt   ru   r<   r   rf   rw   rx   ry   rz   s   @@r1   rJ  rJ  ñ  s,   ù‡ € õ,ð ÷Sñ Só ÷Sð Sr3   rJ  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚEvollaSaProtPooleri  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # r¸   )r;   r<   r   ræ   r?   r  ÚTanhÚ
activationrN   s   &&€r1   r<   ÚEvollaSaProtPooler.__init__  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽr3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r&  r'  )r…   r{   s   "€r1   r†   ÚEvollaSaProtPooler.__annotate__  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r3   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )rV   )rV   r   )r  r`  )rO   rî   Úfirst_token_tensorÚpooled_outputs   &&  r1   rf   ÚEvollaSaProtPooler.forward  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr3   )r`  r  r   rz   s   @@r1   r]  r]    s   ù‡ € õ$÷
÷ ð r3   r]  c                   ó´   a a€ ] tR tRt oR.tRtRtRtRtR]	R]
! ]^RR7      .R]
! ]^R	R7      ./t]P                  ! 4       V 3R
 l4       tV3R ltRtVtV ;t# )ÚEvollaSaProtPreTrainedModeli"  r2  Trî   Ú
attentionsr7  )ÚindexÚ
layer_nameÚcross_attentionsr:  c                óÒ   <€ \         SV `  V4       \        V\        4      '       dA   VP	                  VP
                  4      w  r#\        P                  ! \        VR 4      V4       R# R# )r�   N)	r;   Ú_init_weightsrª   r}   r‹   rP   ÚinitÚcopy_rI   )rO   rÉ   r�   r  rQ   s   &&  €r1   ro  Ú)EvollaSaProtPreTrainedModel._init_weights3  sO   ø€ ä‰Ñ˜fÔ%Ü�fÔ9×:Ò:Ø%×EÑEÀfÇmÁmÓTÑˆMÜ�JŠJ”w˜v zÓ2°MÖBñ ;r3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   r„   )r…   r{   s   "€r1   r†   Ú(EvollaSaProtPreTrainedModel.__annotate__"  ó   ø‡ ‚ áÑò r3   © )rr   rs   rt   ru   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr2  r    rÞ   Ú_can_record_outputsr)   rº   ro  Ú__annotate_func__rw   rx   ry   rz   s   @@r1   ri  ri  "  sƒ   ù‡ € ð -Ð-ÐØÐØ€NØÐØ"&Ðð 	Ð*Ø‘~Ð&?ÀqÐU`ÔaÐbØÙÐ4¸AÐJZÔ[ð
ðÐð ‡]‚]ƒ_ôCó ðC÷% … r3   ri  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR tR t]]R	V3R lR ll4       4       t	Rt
VtV ;t# )
ÚEvollaSaProtProteinEncoderi;  c                ó    <€ V ^8„  d   QhRS[ /# r‚   r„   )r…   r{   s   "€r1   r†   Ú'EvollaSaProtProteinEncoder.__annotate__<  s   ø€ ÷ ñ ™|ñ r3   c                ó¨   <€ \         SV `  V4       \        V4      V n        \	        VR 7      V n        \        V4      V n        V P                  4        R# ©©rP   N)	r;   r<   r5   rb   r}   Úrotary_embeddingsrJ  ÚencoderÚ	post_initrN   s   &&€r1   r<   Ú#EvollaSaProtProteinEncoder.__init__<  s?   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆŒÜ!<ÀFÔ!KˆÔÜ*¨6Ó2ˆŒØ�‰Ör3   c                ó.   € V P                   P                  # r¸   ©rb   rA   ©rO   s   &r1   Úget_input_embeddingsÚ/EvollaSaProtProteinEncoder.get_input_embeddingsC  s   € Ø�‰×.Ñ.Ð.r3   c                ó&   € WP                   n        R # r¸   rŠ  ©rO   rÌ   s   &&r1   Úset_input_embeddingsÚ/EvollaSaProtProteinEncoder.set_input_embeddingsF  s   € Ø*/�‰Ö'r3   c                ó¤   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[S[ P                  ,          S[,          /# )rƒ   r-   Nr_   r”   )r)   r–   r•   r   )r…   r{   s   "€r1   r†   r�  K  sJ   ø€ ÷ !
ñ !
á—<‘< $Õ&ð!
ñ Ÿ™ tÕ+ð!
ñ
 
‰u�|‰|Õ	ÑKÕ	Kñ!
r3   c                óØ  € VP                  4       pVw  rVVP                  pVf   \        P                  ! WV3VR7      pV P	                  WR7      p\        V P                  VVR7      p\        P                  ! WgR7      P                  ^ 4      p	V P                  W‰4      p
V P                  ! V3RVRV
/VB pV^ ,          p\        VVP                  VP                  VP                  R7      # )N©rj   ©r-   r_   )rP   ra   r_   r_   rK   )rV  rî   rj  rm  )rk   rj   r)   Úonesrb   r   rP   rl   rY   r…  r†  r   rî   rj  rm  )rO   r-   r_   rž   rn   Ú
batch_sizeÚ
seq_lengthrj   ra   r`   rK   Úencoder_outputsÚsequence_outputs   &&&,         r1   rf   Ú"EvollaSaProtProteinEncoder.forwardI  sï   € ð  —n‘nÓ&ˆØ!,Ñˆ
à×!Ñ!ˆØÒ!Ü"ŸZšZ¨*Ð)AÈ6ÔRˆNØŸ™°)˜Ó[ˆä2Ø—;‘;Ø'Ø)ô
ˆô —|’| JÔ>×HÑHÈÓKˆØ"×4Ñ4°]ÓQÐàŸ,š,Øñ
Ø*8ð
ØNað
Øekñ
ˆð *¨!Õ,ˆä;Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r3   )rb   r†  r…  r¸   )rr   rs   rt   ru   r<   rŒ  r�  r   r!   rf   rw   rx   ry   rz   s   @@r1   r  r  ;  s?   ù‡ € ÷ó ò/ò0ð  Ø÷!
ñ !
ó ó  ÷!
ð !
r3   r  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )Ú!EvollaSequenceCompressorAttentionio  c                ó„  <€ \         SV `  4        VR,          V n        W0n        W#,          p\        P
                  ! V4      V n        \        P
                  ! V4      V n        \        P                  ! WRR7      V n	        \        P                  ! W^,          RR7      V n
        \        P                  ! WARR7      V n        R# )r  F©ÚbiasNrÓ   )r;   r<   ÚscaleÚheadsr   rC   Ú
norm_mediaÚnorm_latentsræ   Úto_qÚto_kvÚto_out)rO   r&   Údim_headr¢  Ú	inner_dimrQ   s   &&&& €r1   r<   Ú*EvollaSequenceCompressorAttention.__init__p  s�   ø€ Ü‰ÑÔØ˜t•^ˆŒ
ØŒ
ØÕ$ˆ	äŸ,š, sÓ+ˆŒÜŸLšL¨Ó-ˆÔä—I’I˜c°5Ô9ˆŒ	Ü—Y’Y˜s°¥M¸Ô>ˆŒ
Ü—i’i 	°UÔ;ˆŽr3   c                óN  € V P                  V4      pV P                  V4      pV P                  pV P                  V4      p\        P
                  ! W3RR7      pV P                  V4      P                  ^RR7      w  rxVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pVP                  VP                  ^ 4      VP                  ^4      VR4      P                  ^ ^^^4      pWPP                  ,          p\        P                  ! WWP                  RR4      4      p	W™P                  RRR7      P                  4       ,
          p	V	P                   w  r«rÍ\        P"                  ! W¼4      P%                  VP&                  4      pVR,          pVR,          pVV,          pV	P)                  ^V,
          P+                  4       R4      p	V	P-                  RR7      p\        P                  ! VV4      pVP                  ^ ^^^4      pVP/                  VP                  ^ 4      VP                  ^4      R4      pV P1                  V4      # )	z¤
Args:
    x (torch.Tensor): image features
        shape (b, n1, D)
    latent (torch.Tensor): latent features
        shape (b, n2, D);  n2: num of latent tokens
r%   T©r&   ÚkeepdiméþÿÿÿrU   ©rV   NNrV   )NrV   rV   Ng     ˆÃÀ)r£  r¤  r¢  r¥  r)   r®   r¦  Úchunkrõ   rk   Úpermuter¡  rÕ   r­   ÚamaxÚdetachr[   r–  r]   rj   rX   Úboolr×   rø   r§  )rO   r±   Úlatentsr/   ÚhrÁ   Úkv_inputrÂ   ÚvÚsimÚbsÚnhÚskdÚokdr–  Úmask_expÚones_expÚattnÚouts   &&&&               r1   rf   Ú)EvollaSequenceCompressorAttention.forward}  s"  € ð �O‰O˜AÓˆØ×#Ñ# GÓ,ˆà�J‰Jˆà�I‰I�gÓˆÜ—9’9˜a˜\¨rÔ2ˆØ�z‰z˜(Ó#×)Ñ)Ø�2ð *ó 
‰ˆð �F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ—
‘
�Nˆô �lŠl˜1Ÿk™k¨"¨bÓ1Ó2ˆØ—H‘H ¨T�HÓ2×9Ñ9Ó;Õ;ˆØŸ9™9Ñˆ�Ü�zŠz˜"Ó"×%Ñ% d§k¡kÓ2ˆØÐ(Õ)ˆØÐ(Õ)ˆØ˜(Õ"ˆà�o‰o˜q 4�xŸo™oÓ/°Ó6ˆØ�{‰{˜rˆ{Ó"ˆÜ�lŠl˜4 Ó#ˆØ�k‰k˜!˜Q  1Ó%ˆð �k‰k˜#Ÿ(™( 1›+ s§x¡x°£{°BÓ7ˆà�{‰{˜3ÓÐr3   )r¢  r¤  r£  r¡  r¦  r§  r¥  )é@   é   r   rz   s   @@r1   r�  r�  o  s   ù‡ € ÷<÷) ò ) r3   r�  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚEvollaFeedForwardi©  c                ó$  <€ \         SV `  4        \        W,          4      p\        P                  ! V4      V n        \        P                  ! WR R7      V n        \        P                  ! 4       V n	        \        P                  ! W1R R7      V n
        R# ©FrŸ  N)r;   r<   r(   r   rC   Únormræ   Úfc1ÚGELUr`  Úfc2)rO   r&   Úmultr©  rQ   s   &&& €r1   r<   ÚEvollaFeedForward.__init__ª  sZ   ø€ Ü‰ÑÔÜ˜�
“Oˆ	ä—L’L Ó%ˆŒ	Ü—9’9˜S°%Ô8ˆŒÜŸ'š'›)ˆŒÜ—9’9˜Y°%Ô8ˆŽr3   c           	     ó~   € V P                  V P                  V P                  V P                  V4      4      4      4      # r¸   )rÌ  r`  rÊ  rÉ  )rO   r±   s   &&r1   rf   ÚEvollaFeedForward.forward³  s+   € Ø�x‰x˜Ÿ™¨¯©°·±¸1³Ó(>Ó?Ó@Ð@r3   )r`  rÊ  rÌ  rÉ  )é   r   rz   s   @@r1   rÆ  rÆ  ©  s   ù‡ € ÷9÷Aò Ar3   rÆ  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )Ú!EvollaSequenceCompressorResampleri·  c                ó    <€ V ^8„  d   QhRS[ /# r‚   ©r"   )r…   r{   s   "€r1   r†   Ú.EvollaSequenceCompressorResampler.__annotate__¸  s   ø€ ÷ Qñ Q™|ñ Qr3   c                ó´  <€ \         SV `  4        VP                  P                  pVP                  V n        \        P                  ! \        P                  ! V P
                  V4      R R7      V n
        \        P                  ! . 4      V n        \        VP                  4       Fg  pV P                  P                  \        P                  ! \!        W!P"                  VP$                  R7      \'        W!P(                  R7      .4      4       Ki  	  \        P*                  ! VP                  4      V n        \        P.                  ! W!P                  4      V n        R# )T)Úrequires_grad)r&   r¨  r¢  )r&   rÍ  N)r;   r<   Úprotein_encoder_configr?   Úresampler_num_latentsÚnum_latentsr   Ú	Parameterr)   Úrandnrµ  rL  ÚlayersrM  Úresampler_depthÚappendr�  Úresampler_dim_headÚresampler_headsrÆ  Úresampler_ff_multrC   rÉ  ræ   Úprotein_projector)rO   rP   Úprotein_repr_dimr  rQ   s   &&  €r1   r<   Ú*EvollaSequenceCompressorResampler.__init__¸  sö   ø€ Ü‰ÑÔØ!×8Ñ8×DÑDÐØ!×7Ñ7ˆÔÜ—|’|¤E§K¢K°×0@Ñ0@ÐBRÓ$SÐcgÔhˆŒÜ—m’m BÓ'ˆŒÜ�v×-Ñ-Ö.ˆAØ�K‰K×ÑÜ—’ä9Ø 0×;TÑ;TÐ\b×\rÑ\rôô *Ð.>×E]ÑE]Ô^ð	óö	ñ /ô —L’L ×!3Ñ!3Ó4ˆŒ	Ü!#§¢Ð+;×=OÑ=OÓ!PˆÖr3   c                ó’  € VP                   ^ ,          pVP                   w  rE\        P                  ! W@P                  4      P	                  VP
                  4      p\        P                  ! W&3^R7      p\        P                  ! V4      P	                  V P                  P
                  4      pV P                  R,          VP                  R^^4      ,          pVP	                  VP                  4      pV P                   F$  w  ršV	! WV4      V,           pV
! V4      V,           pK&  	  V P                  V4      pV P                  V4      # )r   r%   NrU   )r[   r)   r–  rÛ  r]   rj   r®   rµ  rõ   r^   rÞ  rä  rÉ  )rO   Úembedsr/   Úbrº  r  Úlatent_maskr–  rµ  rÀ  ÚffÚtransformed_features   &&&         r1   rf   Ú)EvollaSequenceCompressorResampler.forwardÍ  sø   € Ø�L‰L˜�Oˆà—
‘
‰ˆÜ—j’j ×%5Ñ%5Ó6×9Ñ9¸$¿+¹+ÓFˆÜ�yŠy˜$Ð,°!Ô4ˆô �zŠz˜!‹}×Ñ §¡× 3Ñ 3Ó4ˆØ—,‘,˜tÕ$ t§y¡y°°Q¸Ó':Õ:ˆØ—*‘*˜VŸ\™\Ó*ˆØŸœ‰HˆDÙ˜6¨DÓ1°GÕ;ˆGÙ˜“k GÕ+ŠGñ $ð #×4Ñ4°WÓ=Ðà�y‰yÐ,Ó-Ð-r3   )rµ  rÞ  rÉ  rÛ  rä  r   rz   s   @@r1   rÓ  rÓ  ·  s   ù‡ € ÷Qó Q÷*.ò .r3   rÓ  c                   ó@   a € ] tR tRt o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚEvollaProteinEncoderModelOutputiá  a>  
sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
    Compressed sequence representation produced by the sequence compressor module. The sequence length is
    reduced from the original input length to `compressed_seq_len` via learned compression.
Nc                óú   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  R3,          R,          ;R&   S[S[ P                  R3,          R,          ;R&   # )rƒ   NÚsequence_compressor_outputrV  .rî   rj  )r)   rñ   r•   )r…   r{   s   "€r1   r†   Ú,EvollaProteinEncoderModelOutput.__annotate__á  sy   ø‡ ‚ ñ !&× 1Ñ 1°DÕ 8Ñ?ñ ñ ×(Ñ(¨4Õ/Ñ6ñ ñ ™×*Ñ*¨CÐ/Õ0°4Õ7Ñ>ñ ñ ‘e×'Ñ'¨Ð,Õ-°Õ4Ñ;ò r3   rv  )rr   rs   rt   ru   rv   rñ  rV  rî   rj  r}  rw   rx   )r{   s   @r1   rï  rï  á  s*   ø‡ € ñð <@ÐØ26ÐØ:>€MØ7;€J÷ ƒ r3   rï  c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )ÚEvollaProteinEncoderið  c                ó    <€ V ^8„  d   QhRS[ /# r‚   rÕ  )r…   r{   s   "€r1   r†   Ú!EvollaProteinEncoder.__annotate__ñ  s   ø€ ÷ ^ñ ^™|ñ ^r3   c                ó|   <€ \         SV `  4        \        VP                  R 7      V n        \        VR 7      V n        R# rƒ  )r;   r<   r  rÙ  ÚmodelrÓ  Úsequence_compressor_resamplerrN   s   &&€r1   r<   ÚEvollaProteinEncoder.__init__ñ  s.   ø€ Ü‰ÑÔÜ/°v×7TÑ7TÔUˆŒ
Ü-NÐV\Ô-]ˆÖ*r3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )rƒ   r-   r_   )r)   Ú
LongTensorrñ   )r…   r{   s   "€r1   r†   rö  ÷  s'   ø€ ÷ 
ñ 
¡×!1Ñ!1ð 
Á5×CTÑCTñ 
r3   c                óŽ   € V P                  WR 7      pVP                  pV P                  WR4      p\        VVP                  R7      # )r•  )rñ  rV  )rø  rV  rù  rï  )rO   r-   r_   rž   Úprotein_outputÚprotein_embedsÚsequence_reprs   &&&,   r1   rf   ÚEvollaProteinEncoder.forwardö  sF   € àŸ™¨i˜ÓWˆØ'×9Ñ9ˆØ×:Ñ:¸>ÓZˆä.Ø'4Ø,×>Ñ>ô
ð 	
r3   )rø  rù  r[  rz   s   @@r1   rô  rô  ð  s*   ù‡ € ÷^ó ^ð
 ÷
ó ÷
ð 
r3   rô  c                   óR   a a€ ] tR tRt oRV3R lV 3R llltR tRR ltRtVtV ;t	# )	Ú#EvollaSequenceAlignerCrossAttentioni  c                óV   <€ V ^8„  d   QhRS[ R,          RS[ R,          RS[ R,          /# )rƒ   Úprotein_encoder_dimNÚstructure_encoder_dimÚmsa_encoder_dim)r(   )r…   r{   s   "€r1   r†   Ú0EvollaSequenceAlignerCrossAttention.__annotate__  s7   ø€ ÷ 1:ñ 1:ñ ! 4�Zð1:ñ  # T�zð	1:ñ
 ˜t�ñ1:r3   c                óÌ  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  R,          V n        \        V P                  V P                  ,          4      V n        V P                  V P                  ,          V n        VP                  pVP                  pVP                  p\        P                  ! V P                  V P                  4      V n        VeL   \        P                  ! W P                  4      V n        \        P                  ! W P                  4      V n        MRV n        RV n        VeL   \        P                  ! W0P                  4      V n        \        P                  ! W0P                  4      V n        MRV n        RV n        VeL   \        P                  ! W@P                  4      V n        \        P                  ! W@P                  4      V n        MRV n        RV n        \)        V P                  4      V n        \        P,                  ! V4      V n        \        P                  ! V P                  V P                  VR7      V n        \3        V P                  V4      V n        \        P6                  ! \8        P:                  ! R.4      4      V n        \        P6                  ! \8        P:                  ! R.4      4      V n        R# )r  NrŸ  rT   rÓ   ) r;   r<   r?   rœ   r¡  r(   rä   rå   Ú$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   ræ   rÊ   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normrF   rH   Úout_projrÆ  rë  rÜ  r)   ÚtensorÚgate_attentionÚgate_ffw)	rO   rP   r  r  r  rç   Úenable_biasÚffn_multrQ   s	   &&&&&   €r1   r<   Ú,EvollaSequenceAlignerCrossAttention.__init__  s  ø€ ô 	‰ÑÔà!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×-Ñ-¨tÕ3ˆŒ
Ü#& t×'7Ñ'7¸$×:RÑ:RÕ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔà'-×'RÑ'RÐ$Ø×0Ñ0ˆØ×*Ñ*ˆä—Y’Y˜t×/Ñ/°×1CÑ1CÓDˆŒ
ØÒ*Ü!ŸyšyÐ)<×>PÑ>PÓQˆDÔÜ!#§¢Ð+>×@RÑ@RÓ!SˆDÕà#ˆDÔØ!%ˆDÔà Ò,Ü!#§¢Ð+@×BTÑBTÓ!UˆDÔÜ#%§9¢9Ð-B×DVÑDVÓ#WˆDÕ à!%ˆDÔØ#'ˆDÔ àÒ&ÜŸ9š9 _×6HÑ6HÓIˆDŒLÜŸYšY ×8JÑ8JÓKˆD�NàˆDŒLØ!ˆDŒNä+¨D×,<Ñ,<Ó=ˆÔä—z’zÐ">Ó?ˆŒäŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ;ÔWˆŒä# D×$4Ñ$4°hÓ?ˆŒÜ Ÿlšl¬5¯<ª<¸¸Ó+>Ó?ˆÔÜŸš¤U§\¢\°3°%Ó%8Ó9ˆŽr3   c	                ó	  € WgV.p	V	 U
u. uF
  qªf   K  V
NK  	  p	p
V	'       g   \        R4      h\        P                  ! V	^R7      p	V P                  V4      pV P	                  V4      pV P
                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpV P                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpV P                  eC   V P                  e5   VP                  V4      pV P                  V4      pV P                  V4      pMRpRpWÎV.pV U
u. uF
  qªf   K  V
NK  	  pp
\        P                  ! V^R7      pWßV.pV U
u. uF
  qªf   K  V
NK  	  pp
\        P                  ! V^R7      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pVP                  4       RR V P                  V P                  3,           pVP                  ! V!  P!                  ^ ^^^4      pW°P"                  ,          pVfO   \        P$                  ! VP                  ^ 4      VP                  ^4      4      P                  VP&                  4      pVR,          V	R,          ,          p\        P(                  ! VVP+                  RR	4      4      pVVP-                  RRR7      P/                  4       ,
          pVP1                  ^V,
          P3                  4       \        P4                  ! VP6                  4      P8                  4      p\:        P<                  ! RR7      ! V4      p\        P(                  ! VV4      pVP!                  ^ ^^^4      P?                  4       pVP                  4       RR	 V P@                  3,           pVP                  ! V!  pV PC                  V4      pV# u up
i u up
i u up
i )
zÁ
query_states: text
key_value_states: protein
query_states: [bs, query_seq_len, dim]
key_value_states: [bs, kv_seq_len, dim]
query_attn_mask: [bs, query_seq_len]
kv_attn_mask: [bs, kv_seq_len]
Nz=At least one modality should be provided for cross attention.r%   Tr¬  rU   )rV   NrV   Nr¯  r®  )"rã   r)   r®   r  rÊ   r  r  r]   r  r  r  r  rk   rœ   rä   rõ   r±  r¡  r–  rj   rÕ   r­   r²  r³  rX   r´  Úfinfor^   Úminr   ÚSoftmaxrØ   rå   r  )rO   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskr  rú   Úkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msarü   rý   Únew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper_   rÚ   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes   &&&&&&&&&                    r1   Úcross_attentionÚ3EvollaSequenceAlignerCrossAttention.cross_attention6  s?  € ð* -ÐFVÐWˆÙ#/ÓA¡<˜aŸ˜¡<ˆÐAßÜÐ\Ó]Ð]Ü—y’y °1Ô5ˆà×)Ñ)¨,Ó7ˆð —j‘j Ó-ˆà×ÑÒ'¨D×,>Ñ,>Ò,JØ'?×'BÑ'BÀ<Ó'PÐ$Ø $× 0Ñ 0Ð1IÓ JÐØ"&×"4Ñ"4Ð5MÓ"NÑà $ÐØ"&Ðà×ÑÒ)¨d×.BÑ.BÒ.NØ)C×)FÑ)FÀ|Ó)TÐ&Ø"&×"4Ñ"4Ð5OÓ"PÐØ$(×$8Ñ$8Ð9SÓ$TÑ!à"&ÐØ$(Ð!à�<‰<Ò#¨¯©Ò(BØ#7×#:Ñ#:¸<Ó#HÐ Ø ŸL™LÐ)=Ó>ˆMØ"Ÿn™nÐ-AÓB‰Oà ˆMØ"ˆOà&¸]ÐKˆ	Ù )Ó;¡	˜1—Q�Q¡	ˆ	Ð;Ü—I’I˜i¨QÔ/ˆ	à*À?ÐSˆÙ"-Ó?¡+˜Q—q�q¡+ˆÐ?Ü—i’i °Ô3ˆà +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
õ !
Ðð "×&Ò&Ð(=Ñ>×FÑFÀqÈ!ÈQÐPQÓRˆà'Ÿn™nÓ.¨s°Ð3Ø×$Ñ$Ø×$Ñ$ð7
õ 
Ðð —N’NÐ$7Ñ8×@Ñ@ÀÀAÀqÈ!ÓLˆ	à +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
õ !
Ðð "×&Ò&Ð(=Ñ>×FÑFÀqÈ!ÈQÐPQÓRˆà!§J¡JÕ.ˆð Ò"Ü#Ÿjšj¨×):Ñ):¸1Ó)=¸|×?PÑ?PÐQRÓ?SÓT×WÑWÐXd×XkÑXkÓlˆOØ(Ð)9Õ:¸\ÐJZÕ=[Õ[ˆä—|’| K°×1DÑ1DÀRÈÓ1LÓMˆØ# l×&7Ñ&7¸BÈÐ&7Ó&M×&TÑ&TÓ&VÕVˆØ'×3Ñ3Ø�Õ×%Ñ%Ó'¬¯ª°\×5GÑ5GÓ)H×)LÑ)Ló
Ðô Ÿ*š*¨Õ,Ð-=Ó>ˆô Ÿš _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÕ"SÐØ%×*Ò*Ð,CÑDˆàŸ™ mÓ4ˆàÐùòq BùòL <ùò @s"   ‰Q5“Q5Å,Q:Å6Q:ÆQ?Æ(Q?c                óâ  € Ve   VP                   w  rÞpVfk   \        P                  ! WÞ4      P                  V	P                  4      V	P                  Wí3R7      P                  ,          P                  VP                  4      pMR pVe‚   VP                   w  pppVfm   \        P                  ! VV4      P                  V	P                  4      V
P                  VV3R7      P                  ,          P                  VP                  4      pMR pVe‚   VP                   w  pppVfm   \        P                  ! VV4      P                  V	P                  4      VP                  VV3R7      P                  ,          P                  VP                  4      pMR pTpVe   VP                  4       '       g5   Ve   VP                  4       '       g   Ve£   VP                  4       '       d�   TpV P                  VVVVVVVVR7      p\        P                  ! V P                  4      V,          pVV,           pTpV P                  V4      \        P                  ! V P                  4      ,          pVV,           pV# )N)rk   )r   r!  r"  r#  r$  r%  r&  r'  )r[   r)   r–  r]   rj   rm   ÚTÚanyr6  Útanhr  rë  r  )rO   r   Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesr$  r%  r&  r'  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesrº  Úprotein_kv_seq_lenr&   Ústructure_kv_seq_lenÚmsa_kv_seq_lenrî   Úresiduals   &&&&&&&&&&&&&       r1   rf   Ú+EvollaSequenceAlignerCrossAttention.forward¦  sP  € ð Ò(Ø*;×*AÑ*AÑ'ˆB CØ#Ò+ä—J’J˜rÓ6×9Ñ9Ð:L×:SÑ:SÓTØ(×/Ñ/Ð6HÐ5MÐ/ÓN×PÑPõQç‘"Ð&×-Ñ-Ó.ð %øð
 $(Ð àÒ*Ø,?×,EÑ,EÑ)ˆBÐ$ cØ%Ò-ä—J’J˜rÐ#7Ó8×;Ñ;Ð<N×<UÑ<UÓVØ*×1Ñ1Ð8LÈbÐ7QÐ1ÓR×TÑTõUç‘"Ð(×/Ñ/Ó0ð 'øð
 &*Ð"àÒ$Ø&3×&9Ñ&9Ñ#ˆB� ØÒ'ä—J’J˜r >Ó2×5Ñ5Ð6H×6OÑ6OÓPØ$×+Ñ+°.À"Ð1EÐ+ÓF×HÑHõIç‘"�]×)Ñ)Ó*ð !øð
  $ÐØ$ˆð Ò*Ð/C×/GÑ/G×/IÒ/IØ#Ò/Ð4J×4NÑ4N×4PÒ4PØÒ)Ð.>×.BÑ.B×.DÒ.Dà$ˆHØ ×0Ñ0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ó 	ˆMô "ŸJšJ t×':Ñ':Ó;¸mÕKˆMà$ }Õ4ˆMà$ˆHØ ŸG™G MÓ2´U·Z²ZÀÇÁÓ5NÕNˆMØ$ }Õ4ˆMàÐr3   )rå   rä   r  rH   rë  r  r  r?   r  r  r  rœ   r  rÊ   r¡  r  r  r  ©NNN)NNNNNNN)
rr   rs   rt   ru   r<   r6  rf   rw   rx   ry   rz   s   @@r1   r  r    s#   ù‡ € ÷1:õ 1:òfn÷`Gô Gr3   r  ÚRMSNormc                   óZ   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltR tRtVtV ;t	# )	r  ið  c                ó$   <€ V ^8„  d   QhRS[ RR/# )rƒ   r8   r”   N)r\   )r…   r{   s   "€r1   r†   ÚEvollaRMSNorm.__annotate__ò  s   ø€ ÷ $ñ $©ð $¸$ñ $r3   c                óŽ   <€ \         SV `  4        \        P                  ! \        P
                  ! V4      4      V n        W n        R# )z,
EvollaRMSNorm is equivalent to T5LayerNorm
N)r;   r<   r   rÜ  r)   r–  ÚweightÚvariance_epsilon)rO   r?   r8   rQ   s   &&&€r1   r<   ÚEvollaRMSNorm.__init__ò  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Ör3   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r&  r'  )r…   r{   s   "€r1   r†   rL  ú  s#   ø€ ÷ ;ñ ;¡U§\¡\ð ;±e·l±lñ ;r3   c                ó:  € VP                   pVP                  \        P                  4      pVP	                  ^4      P                  RRR7      pV\        P                  ! W0P                  ,           4      ,          pV P                  VP                  V4      ,          # )rƒ   T)r­  rU   )	r^   r]   r)   Úfloat32ÚpowÚmeanÚrsqrtrO  rN  )rO   rî   Úinput_dtypeÚvariances   &&  r1   rf   ÚEvollaRMSNorm.forwardú  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯ª°H×?TÑ?TÕ4TÓ(UÕUˆØ�{‰{˜]×-Ñ-¨kÓ:Õ:Ð:r3   c                ó^   € \        V P                  P                  4       R V P                   2# )z, eps=)r•   rN  r[   rO  r‹  s   &r1   Ú
extra_reprÚEvollaRMSNorm.extra_repr  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr3   )rO  rN  )g�íµ ÷Æ°>)
rr   rs   rt   ru   r<   rf   r[  rw   rx   ry   rz   s   @@r1   r  r  ð  s%   ù‡ € ÷$õ $÷;ð ;÷Jò Jr3   r  c                   óÎ   a a€ ] tR tRt o]! RRR7      RV3R lV 3R lll4       t]]! RRR7      RV3R lR ll4       4       t]P                  ! 4       ]
R	 4       4       tR
tVtV ;t# )ÚEvollaRotaryEmbeddingi  rj   r~   r   c                ó    <€ V ^8„  d   QhRS[ /# r‚   rÕ  )r…   r{   s   "€r1   r†   Ú"EvollaRotaryEmbedding.__annotate__  s   ø€ ÷ Oñ O™|ñ Or3   c                óä  <€ \         SV `  4        VP                  V n        VP                  V n        Wn        V P
                  P                  R ,          V n        V P                  pV P                  R8w  d   \        V P                  ,          pV! V P
                  V4      w  q@n
        \        P                  ! VRR7      V n        \        P                  ! VP                  4       RR7      V n        R# )rŠ   ÚdefaultF)Ú
persistentN)r;   r<   rJ   Úmax_seq_len_cachedÚoriginal_max_seq_lenrP   Úrope_parametersrŠ   r‹   r   r‰   r   rŒ   r�   ÚcloneÚoriginal_inv_freq)rO   rP   rj   Úrope_init_fnr�   rQ   s   &&&  €r1   r<   ÚEvollaRotaryEmbedding.__init__  s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÕAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÔ&Ü.¨t¯~©~Õ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÔ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÔ!NˆÖr3   c                óP   <€ V ^8„  d   QhRS[ RS[S[P                  S[3,          /# r“   )r"   r•   r)   r–   r\   )r…   r{   s   "€r1   r†   r`    s.   ø€ ÷ 5ñ 5±ð 5ÑX]Ñ^c×^jÑ^jÑlqÐ^qÕXrñ 5r3   c           	     ó0  € V P                   R,          p\        V RR4      ;'       g    V P                  V P                  ,          pRpRV\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          pVP                  V4      V3# )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
r›   r˜   Nr™   rš   )rf  rI   r?   rœ   r)   rl   r\   r]   r�   s   &&,    r1   r‹   Ú5EvollaRotaryEmbedding.compute_default_rope_parameters  s~   € ð ×%Ñ% lÕ3ˆÜ�f˜j¨$Ó/×cÐc°6×3EÑ3EÈ×IcÑIcÕ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÔ#MÐPSÕ#SÕTÕUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r3   c                ó^  € V P                   R,          P                  VP                  ^ ,          R	^4      P                  \        P
                  VP                  R7      pVR
,          P                  4       p\        VP                  P                  \        4      '       d2   VP                  P                  R8w  d   VP                  P                  MRp\        VRR7      ;_uu_ 4        W4,          P                  ^^4      p\        P                  ! Wf3R	R7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)Nri   r£   r¤   Fr¥   r%   rš   r¨   rU   r©   )r�   rm   r[   r]   r)   r\   rj   rª   r«   r¬   r   r­   r®   r¯   r‰   r°   r^   )
rO   r±   r`   r³   r´   r¦   rµ   r¶   r¯   r°   s
   &&&       r1   rf   ÚEvollaRotaryEmbedding.forward+  s2  € ð �M‰M˜-Õ(×/Ñ/°×0BÑ0BÀ1Õ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÓtð 	ð !-¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆä¨¸U×CÖCØ&Õ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   Ã(A4FÆF,	)r‰   rP   r�   rd  rh  re  rŠ   r¸   )rr   rs   rt   ru   r   r<   r¹   r‹   r)   rº   r   rf   rw   rx   ry   rz   s   @@r1   r^  r^    su   ù‡ € Ù�X vÔ.÷Oô Oó /ðOð  Ù�X vÔ.÷5ñ 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ÷<ð <r3   r^  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú	EvollaMLPi>  c                ó  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! V P                  V P                  VP                  R 7      V n        \
        P                  ! V P                  V P                  VP                  R 7      V n	        \
        P                  ! V P                  V P                  VP                  R 7      V n
        \        VP                  ,          V n        R# )rŸ  N)r;   r<   rP   r?   r#  r   ræ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fnrN   s   &&€r1   r<   ÚEvollaMLP.__init__?  s¶   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Õ/ˆŽr3   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r¸   )rv  rx  rt  ru  )rO   r±   rv  s   && r1   rf   ÚEvollaMLP.forwardI  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr3   )rx  rP   rv  rt  r?   r#  ru  r   rz   s   @@r1   rq  rq  >  s   ù‡ € õ0÷ò r3   rq  c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )rƒ   rî   rÙ   r”   )r)   r–   r(   )r…   s   "r1   r†   r†   N  s.   € ÷ 	Uñ 	UœUŸ\™\ð 	U´#ð 	U¼%¿,¹,ñ 	Ur3   c                ó˜   € V P                   w  r#rEV^8X  d   V # V R,          P                  W#WV4      p V P                  W#V,          WE4      # )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
)rV   rV   NrV   rV   )r[   rm   rø   )rî   rÙ   ÚbatchÚnum_key_value_headsÚslenr˜   s   &&    r1   rÔ   rÔ   N  sU   € ð
 2?×1DÑ1DÑ.€E Ø�„zØÐØ!Ð"2Õ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÕ(CÀTÓTÐTr3   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚEvollaAttentioniZ  z=Multi-headed attention from 'Attention Is All You Need' paperc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rƒ   rP   ré   ©r"   r(   )r…   r{   s   "€r1   r†   ÚEvollaAttention.__annotate__^  s   ø€ ÷ 
ñ 
™|ð 
¹ñ 
r3   c                ó¢  <€ \         SV `  4        Wn        W n        \	        VR VP
                  VP                  ,          4      V n        VP                  VP                  ,          V n	        V P                  R,          V n
        VP                  V n        RV n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP
                  VP                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  ,          VP
                  VP                  R7      V n        R# )r˜   TrŸ  NrÓ   )r;   r<   rP   ré   rI   r?   rœ   r˜   r  rÐ   rÍ   Úattention_dropoutrê   r   ræ   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©rO   rP   ré   rQ   s   &&&€r1   r<   ÚEvollaAttention.__init__^  sI  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÕ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Õ$\ˆÔ!Ø—}‘} dÕ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Õ JÐQW×QfÑQfô
ˆŒô —i’iØ×&Ñ&¨¯©Õ6¸×8JÑ8JÐQW×QfÑQfô
ˆŽr3   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  3,          /# )rƒ   rî   rK   Nr_   rB  rž   r”   )r)   r–   r•   r   r   r   )r…   r{   s   "€r1   r†   r†  u  s‡   ø€ ÷ &)ñ &)á—|‘|ð&)ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð&)ñ Ÿ™ tÕ+ð	&)ñ
  �ð&)ñ Ñ+Õ,ð&)ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ&)r3   c                ó  € VP                   R R p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      pV P                  V4      P                  V4      P	                  ^^4      p	V P                  V4      P                  V4      P	                  ^^4      p
Vw  r¼\        W‰W¼4      w  r‰Ve   VP                  WšV P                  4      w  rš\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                  '       g   RMV P                   RV P"                  /VB w  rïVP$                  ! . VORN5!  P'                  4       pV P)                  V4      pWï3# )NrH   rT   rÍ   rU   )r[   r˜   rŠ  rõ   r­   r‹  rŒ  rÇ   Úupdateré   r   rö   rP   r÷   rÜ   rÒ   rˆ  rÍ   rø   rØ   r�  )rO   rî   rK   r_   rB  rž   rn   rù   r   Ú
key_statesÚvalue_statesr¯   r°   rþ   rÛ   rÚ   s   &&&&&,          r1   rf   ÚEvollaAttention.forwardu  s‡  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÒ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r3   )rˆ  rP   r˜   rê   r‹  ré   rÐ   r�  rŠ  rÍ   rŒ  rH  )
rr   rs   rt   ru   rv   r<   rf   rw   rx   ry   rz   s   @@r1   r‚  r‚  Z  s!   ù‡ € áG÷
ó 
÷.&)÷ &)ò &)r3   r‚  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚEvollaDecoderLayeriž  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r„  r…  )r…   r{   s   "€r1   r†   ÚEvollaDecoderLayer.__annotate__Ÿ  s   ø€ ÷ ñ ™|ð ¹ñ r3   c                óÐ  <€ \         SV `  4        VP                  V n        \        WR 7      V n        \        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        V^,           \        VP                  VP                  ,          ^4      ,          ^ 8X  d   \        VVP                  R7      V n        R# R# )©rP   ré   r7   )r  N)r;   r<   r?   r‚  Ú	self_attnrq  Úmlpr  Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚmaxrN  Úaligner_num_add_layersr  ÚadapterrŽ  s   &&&€r1   r<   ÚEvollaDecoderLayer.__init__Ÿ  s³   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä(°ÔLˆŒä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%Ø˜�MœS ×!9Ñ!9¸V×=ZÑ=ZÕ!ZÐ\]Ó^Õ^ÐbcÔcÜ>ØØ$*×$6Ñ$6ôˆDŽLñ dr3   c                ó0  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  /# )rƒ   rî   rK   Nr_   r`   rB  Ú	use_cacher<  r=  r>  r?  r@  rA  r$  r”   )r)   r–   r•   rü  r   r´  )r…   r{   s   "€r1   r†   r™  ®  s  ø€ ÷ 3ñ 3á—|‘|ð3ñ #¡5§<¡<±·±Ð#=Õ>ÀÕEð3ñ Ÿ™ tÕ+ð	3ñ
 ×&Ñ&¨Õ-ð3ñ  �ð3ñ ˜$•;ð3ñ !Ÿ<™<¨$Õ.ð3ñ #Ÿ\™\¨DÕ0ð3ñ —|‘| dÕ*ð3ñ "ŸL™L¨4Õ/ð3ñ $Ÿl™l¨TÕ1ð3ñ Ÿ™ tÕ+ð3ñ Ÿ™¨Õ,ð3ñ  
�‰ñ!3r3   c                ó,  € TpV P                  V4      pV P                  ! RR VRVRVRVRVRV/VB w  ppWñ,           pTpV P                  V4      pV P                  V4      pWñ,           p\	        V R4      '       d   V P                  VVVV	VV
VVR7      pV# )	rî   r_   r`   rB  r¦  rK   r£  )r   r<  r=  r>  r$  r?  r@  rA  rv  )rŸ  rœ  r   r�  râ   r£  )rO   rî   rK   r_   r`   rB  r¦  r<  r=  r>  r?  r@  rA  r$  rž   rF  r  s   &&&&&&&&&&&&&&,  r1   rf   ÚEvollaDecoderLayer.forward®  sç   € ð" !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ñ 
Ø'ð
à)ð
ð &ð
ð ,ð	
ð
  ð
ð !4ð
ð ñ
Ñˆ�qð !Õ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Õ0ˆä�4˜×#Ò#Ø ŸL™LØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ó 	ˆMð Ðr3   )r£  r?   rŸ  r�  r   rœ  )NNNNFNNNNNNNr   rz   s   @@r1   r—  r—  ž  s   ù‡ € ÷ó ÷3÷ 3ò 3r3   r—  c                   óœ   a a€ ] tR tRt oRtRt. ROtR.tRtRt	Rt
RtRtR]R]/t]P"                  ! 4       V 3R l4       tV3R	 ltR
tVtV ;t# )ÚEvollaPreTrainedModeliä  rø  TrB  Frî   rj  c                óÌ  <€ V P                   P                  p\        SV `  V4       \	        V\
        4      '       dm   \        P                  ! VP                  4       \        P                  ! VP                  4       \        P                  ! VP                  P                  4       R# \	        V\        4      '       d&   \        P                  ! VP                  R VR7       R# R# )rT   )rU  ÚstdN)rP   Úinitializer_ranger;   ro  rª   r  rp  Úzeros_r  r  Úones_r  rN  rÓ  Únormal_rµ  )rO   rÉ   r¬  rQ   s   && €r1   ro  Ú#EvollaPreTrainedModel._init_weightsû  s�   ø€ à�k‰k×+Ñ+ˆÜ‰Ñ˜fÔ%Ü�fÔA×BÒBÜ�KŠK˜×-Ñ-Ô.Ü�KŠK˜Ÿ™Ô(Ü�JŠJ�v×,Ñ,×3Ñ3Ö4Ü˜Ô A×BÒBÜ�LŠL˜Ÿ™¨c°s×;ñ Cr3   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r‚   rÕ  )r…   r{   s   "€r1   r†   Ú"EvollaPreTrainedModel.__annotate__ä  ru  r3   rv  )r—  r2  rÓ  r  )rr   rs   rt   ru   Úbase_model_prefixÚsupports_gradient_checkpointingrw  Ú_skip_keys_device_placementrx  ry  rz  Ú_can_compile_fullgraphr{  r—  r‚  r|  r)   rº   ro  r}  rw   rx   ry   rz   s   @@r1   rª  rª  ä  sw   ù‡ € ð  ÐØ&*Ð#òÐð $5Ð"5ÐØ ÐØ€NØÐà!ÐØ"'ÐàÐ+Ø�oðÐð
 ‡]‚]ƒ_ô<ó ð<÷1 … r3   rª  c                   ó~   a a€ ] tR tRt oV3R lV 3R lltR tR t]]]	R	V3R lR ll4       4       4       t
RtVtV ;t# )
ÚEvollaModeli  c                ó    <€ V ^8„  d   QhRS[ /# r‚   rÕ  )r…   r{   s   "€r1   r†   ÚEvollaModel.__annotate__  s   ø€ ÷ ñ ™|ñ r3   c           
     óP  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! V P                  VP                  V P                  4      V n        \        VR 7      V n
        \
        P                  ! \        VP                  4       Uu. uF  p\        VVR7      NK  	  up4      V n        \!        VP                  VP"                  R7      V n        \'        VRR4      V n        \+        VR 7      V n        V P/                  4        R# u upi )r„  r›  r7   rQ  FN)r;   r<   r@   r.   r>   r   r=   r?   Úembed_tokensrô  Úprotein_encoderrL  rM  rN  r—  rÞ  r  rž  rÉ  rI   rQ  r^  Ú
rotary_embr‡  rŽ  s   && €r1   r<   ÚEvollaModel.__init__  sð   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨¯©¸&×:LÑ:LÈd×N^ÑN^Ó_ˆÔÜ3¸6ÔBˆÔÜ—m’mô "' v×'?Ñ'?Ô!@óñ
 "A�Iô	 #Ø!Ø'÷ñ "Añó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü&-¨fÐ6NÐPUÓ&VˆÔ#Ü/°vÔ>ˆŒØ�‰Öùòs   Â&D#c                ó   € V P                   # r¸   ©r½  r‹  s   &r1   rŒ  Ú EvollaModel.get_input_embeddings  s   € Ø× Ñ Ð r3   c                ó   € Wn         R # r¸   rÂ  r�  s   &&r1   r�  Ú EvollaModel.set_input_embeddings   s   € Ø!Ör3   c                óè  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          /# )rƒ   r-   Nr_   r`   rB  ra   r¦  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsr@  rA  r”   )r)   rü  r–   r   rñ   r´  r•   r   )r…   r{   s   "€r1   r†   r»  &  s  ø€ ÷ ]ñ ]á×#Ñ# dÕ*ð]ñ Ÿ™ tÕ+ð]ñ ×&Ñ&¨Õ-ð	]ñ
  �ð]ñ ×(Ñ(¨4Õ/ð]ñ ˜$•;ð]ñ !×+Ñ+¨dÕ2ð]ñ !&§¡¨tÕ 3ð]ñ ×*Ñ*¨TÕ1ð]ñ ×$Ñ$ tÕ+ð]ñ $Ÿl™l¨TÕ1ð]ñ Ÿ™ tÕ+ð]ñ 
Ñ(Õ	(ñ]r3   c                óT  € VRJ VRJ,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      pRpRpVef   Veb   V P                  VVR7      pVP                  p\
        P                  ! VP                  ^ ,          VP                  \
        P                  R7      p\        V P                  VVVR7      pTpV P                  VVR7      pV P                    F$  pV! V3R	VR
VRVRVRVRV	RV
RVRVRVRVRV/VB pK&  	  V P#                  V4      p\%        VVR7      pV# )aÑ  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
structure_feats (torch.FloatTensor):
    The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
msa_feats (torch.FloatTensor):
    The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
structure_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummy input for now.
msa_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummy input for now.
Nz:You must specify exactly one of input_ids or inputs_embedsr„  r”  r•  )rj   r^   )rP   ra   r_   rB  )r`   r_   r`   rB  r¦  r<  r=  r>  r?  r@  rA  r$  rK   )rV  rB  )rã   r½  r	   rP   Úget_seq_lengthr)   rl   r[   rj   rY   r¾  rñ  r–  r´  r   r¿  rÞ  rÉ  r   )rO   r-   r_   r`   rB  ra   r¦  rÇ  rÈ  rÉ  rÊ  r@  rA  rž   Úpast_seen_tokensÚprotein_featsr?  Úprotein_outputsÚcausal_maskrî   rK   Údecoder_layerr  s   &&&&&&&&&&&&&,         r1   rf   ÚEvollaModel.forward#  s
  € ðB ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLàˆØ!ÐàÒ(Ð-CÒ-OØ"×2Ñ2Ø+Ø5ð 3ó ˆOð ,×FÑFˆMÜ!&§¢Ø!×'Ñ'¨Õ*Ø(×/Ñ/Ü—j‘jô"Ðô )Ø—;‘;Ø'Ø)Ø+ô	
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[œ[ˆMÙ)Øñà*ðð *ðð !0ð	ð
 $ðð #0ðð %4ðð (ðð $6ðð &:ðð  .ðð !/ðð %8ðð ñŠMñ )ð$ Ÿ	™	 -Ó0ˆä(Ø+Ø+ô
ˆð ˆr3   )r½  rQ  rÞ  rÉ  r.   r¾  r¿  r>   )NNNNNNNNNNNN)rr   rs   rt   ru   r<   rŒ  r�  r   r   r!   rf   rw   rx   ry   rz   s   @@r1   r¹  r¹    sK   ù‡ € ÷ó ò*!ò"ð ØØ÷]ñ ]ó ó  ó ÷]ð ]r3   r¹  c                   óh   a a€ ] tR tRt oV 3R ltR tR t]]RV3R lR ll4       4       t	Rt
VtV ;t# )	ÚEvollaForProteinText2Texti†  c                óì   <€ \         SV `  V4       \        V4      V n        VP                  V n        \
        P                  ! VP                  V P                  R R7      V n        V P                  4        R# rÈ  )
r;   r<   r¹  rø  r>   r   ræ   r?   Úlm_headr‡  rN   s   &&€r1   r<   Ú"EvollaForProteinText2Text.__init__‡  sQ   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°T·_±_È5ÔQˆŒà�‰Ör3   c                ó6   € V P                   P                  4       # r¸   )rø  rŒ  r‹  s   &r1   rŒ  Ú.EvollaForProteinText2Text.get_input_embeddings�  s   € Ø�z‰z×.Ñ.Ó0Ð0r3   c                ó8   € V P                   P                  V4      # r¸   )rø  r�  r�  s   &&r1   r�  Ú.EvollaForProteinText2Text.set_input_embeddings’  s   € Ø�z‰z×.Ñ.¨uÓ5Ð5r3   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          /# )
rƒ   r-   Nr_   ra   ÚlabelsrÇ  rÈ  r¦  Úlogits_to_keep)r)   rü  r–   rñ   r´  r(   )r…   r{   s   "€r1   r†   Ú&EvollaForProteinText2Text.__annotate__—  s°   ø€ ÷ Bñ Bá×#Ñ# dÕ*ðBñ Ÿ™ tÕ+ðBñ ×(Ñ(¨4Õ/ð	Bñ
 × Ñ  4Õ'ðBñ !×+Ñ+¨dÕ2ðBñ !&§¡¨tÕ 3ðBñ ˜$•;ðBñ ™eŸl™lÕ*ñBr3   c	                ó†  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  p\        V\        4      '       d   \	        V) R4      MTpV P                  VRVR3,          4      pRpVe#   V P                  ! RR	VR
VRV P                  /V	B p\        VVV
P                  V
P                  V
P                  R7      pV# )a|  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

Example:

```python
>>> from transformers import EvollaProcessor, EvollaForProteinText2Text
>>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
>>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

>>> protein_information = {
    "aa_seq": "your amino acid sequence",
    "foldseek": "your foldseek sequence",
}
>>> question = "What is the function of this protein?"
>>> message = [
    {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
    {"role": "user", "content": question},
]

>>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
>>> outputs = model.generate(**inputs)

>>> print(processor.batch_decode(outputs, skip_special_tokens=True))
```r-   r_   ra   rÇ  rÈ  r¦  NrV   ÚlogitsrÝ  r>   )Úlossrá  rB  rî   rj  rv  )rø  rV  rª   r(   ÚslicerÖ  Úloss_functionr>   r   rB  rî   rj  )rO   r-   r_   ra   rÝ  rÇ  rÈ  r¦  rÞ  rž   Úoutputsrî   Úslice_indicesrá  râ  Ú
lm_outputss   &&&&&&&&&,      r1   rf   Ú!EvollaForProteinText2Text.forward•  sÿ   € ðT ,0¯:ª:ñ ,
Øð,
à)ð,
ð (ð,
ð 0ð	,
ð
 $:ð,
ð  ð,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%Ñi¨VÐi¸FÐiÈtÏÉÐiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ˆ
ð Ðr3   )rÖ  rø  r>   )NNNNNNNr   )rr   rs   rt   ru   r<   rŒ  r�  r   r   rf   rw   rx   ry   rz   s   @@r1   rÔ  rÔ  †  s>   ù‡ € õò1ò6ð Ø÷Bñ Bó ó ÷Bð Br3   rÔ  )r4  )NrT   )rÔ  r¹  rª  )Xr  Úcollections.abcr   Údataclassesr   r)   r   Ú r   rp  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr    r!   Úconfiguration_evollar"   r#   r2   rÎ   r5   r}   r¾   rÇ   rÜ   rÞ   r  r  r  r!  r,  r2  rJ  r]  ri  r  r�  rÆ  rÓ  rï  rô  r  r  r^  rq  rÔ   r‚  r—  rª  r¹  rÔ  Ú__all__rv  r3   r1   Ú<module>rü     sµ  ðó* Ý $Ý !ã Ý å &Ý !ß .Ý )ß Lß JÝ 9÷õ ÷ Lß FÝ &ß IÑ IÝ 0ß Gß Eß <ò4ô Y=˜RŸY™Yô Y=ôx6< "§)¡)ô 6<òr(ñ Ð-Ó.óBó /ðB÷4 %ñF Ð)Ó*ôK) §	¡	ó K)ó +ðK)ô\
˜RŸY™Yô 
ô˜BŸI™Iô ò<;ô˜rŸy™yô ô
˜Ÿ™ô 
ô7Ð2ô 7ôtS˜"Ÿ)™)ô SôD˜Ÿ™ô ð ôC /ó Có ðCô01
Ð!<ô 1
ôh7 ¨¯	©	ô 7 ôtA˜Ÿ	™	ô Aô'.¨¯	©	ô '.ðT Ø
ô
< kó 
<ó ó ð
<ô
˜2Ÿ9™9ô 
ô$k¨"¯)©)ô kñ\ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(6<˜BŸI™Iô 6<ôr�—	‘	ô õ 	Uñ Ð)Ó*ô@)�b—i‘ió @)ó +ð@)ôFCÐ3ô CðL ô<˜Oó <ó ð<ôD|Ð'ô |ô~SÐ 5°ô Sòl P‚r3   