+
    œ°³jnM ã                   ó‚  € ^ RI t ^ RIt^ RIHtHt ^ RIHt ^ RIHt ^ RI	t
^ RIt^ RIHt ^ RIHt ^RIHt ^RIHt ^RIHt ^R	IHtHtHt ^R
IHt ^RIHtHt ^RIH t H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t)H*t*H+t+ ]PX                  ! ]-4      t.]! RR7      ] ! R R]4      4       4       t/R t0R t1RMR R llt2R R lt3R R lt4R t5 ! R R]Pl                  4      t7 ! R R]Pp                  4      t9]Pt                  Pv                  RNR R ll4       t< ! R  R!]Pp                  4      t= ! R" R#]Pp                  4      t> ! R$ R%]Pp                  4      t? ! R& R']4      t@ ! R( R)]Pp                  4      tA ! R* R+]Pp                  4      tB ! R, R-]Pp                  4      tC ! R. R/]Pp                  4      tD ! R0 R1]Pp                  4      tE ! R2 R3]Pp                  4      tF ! R4 R54      tGROR6 ltHR7 tI ! R8 R9]Pp                  4      tJ ! R: R;]Pp                  4      tK ! R< R=]Pp                  4      tL ! R> R?]Pp                  4      tM ! R@ RA]Pp                  4      tN ! RB RC]Pp                  4      tO ! RD RE]Pp                  4      tP ! RF RG]Pp                  4      tQ ! RH RI]Pp                  4      tR]! RJR7       ! RK RL]4      4       tSRLR'.tTR# )Pé    N)ÚCallableÚSequence)Ú	dataclass)Úpartial)Ú	LayerNorm)Úinitialization)Úis_deepspeed_available)ÚModelOutput)ÚContextManagersÚauto_docstringÚlogging)Úmaybe_autocast)ÚEsmModelÚEsmPreTrainedModel)Ú	OFProteinÚRigidÚRotationÚatom14_to_atom37Úchunk_layerÚcompute_predicted_aligned_errorÚ
compute_tmÚ-frames_and_literature_positions_to_atom14_posÚmake_atom14_masksÚresidue_constantsÚto_pdbÚtorsion_angles_to_framesz8
    Output type of [`EsmForProteinFoldingOutput`].
    )Úcustom_introc                   óŒ   a € ] tR t^6t o RtRtRtRtRtRt	Rt
RtRtRtRtRtRtRtRtRtRtRtRtRtRtRtRtRtV 3R ltRtV tR# )ÚEsmForProteinFoldingOutputa_	  
frames (`torch.FloatTensor`):
    Output frames.
sidechain_frames (`torch.FloatTensor`):
    Output sidechain frames.
unnormalized_angles (`torch.FloatTensor`):
    Predicted unnormalized backbone and side chain torsion angles.
angles (`torch.FloatTensor`):
    Predicted backbone and side chain torsion angles.
positions (`torch.FloatTensor`):
    Predicted positions of the backbone and side chain atoms.
states (`torch.FloatTensor`):
    Hidden states from the protein folding trunk.
s_s (`torch.FloatTensor`):
    Per-residue embeddings derived by concatenating the hidden states of each layer of the ESM-2 LM stem.
s_z (`torch.FloatTensor`):
    Pairwise residue embeddings.
distogram_logits (`torch.FloatTensor`):
    Input logits to the distogram used to compute residue distances.
lm_logits (`torch.FloatTensor`):
    Logits output by the ESM-2 protein language model stem.
aatype (`torch.FloatTensor`):
    Input amino acids (AlphaFold2 indices).
atom14_atom_exists (`torch.FloatTensor`):
    Whether each atom exists in the atom14 representation.
residx_atom14_to_atom37 (`torch.FloatTensor`):
    Mapping between atoms in the atom14 and atom37 representations.
residx_atom37_to_atom14 (`torch.FloatTensor`):
    Mapping between atoms in the atom37 and atom14 representations.
atom37_atom_exists (`torch.FloatTensor`):
    Whether each atom exists in the atom37 representation.
residue_index (`torch.FloatTensor`):
    The index of each residue in the protein chain. Unless internal padding tokens are used, this will just be
    a sequence of integers from 0 to `sequence_length`.
lddt_head (`torch.FloatTensor`):
    Raw outputs from the lddt head used to compute plddt.
plddt (`torch.FloatTensor`):
    Per-residue confidence scores. Regions of low confidence may indicate areas where the model's prediction is
    uncertain, or where the protein structure is disordered.
ptm_logits (`torch.FloatTensor`):
    Raw logits used for computing ptm.
ptm (`torch.FloatTensor`):
    TM-score output representing the model's high-level confidence in the overall structure.
aligned_confidence_probs (`torch.FloatTensor`):
    Per-residue confidence scores for the aligned structure.
predicted_aligned_error (`torch.FloatTensor`):
    Predicted error between the model's prediction and the ground truth.
max_predicted_aligned_error (`torch.FloatTensor`):
    Per-sample maximum predicted error.
Nc                ó<  <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R	&   S[ P                  R,          ;R
&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[ P                  R,          ;R&   # )é   NÚframesÚsidechain_framesÚunnormalized_anglesÚanglesÚ	positionsÚstatesÚs_sÚs_zÚdistogram_logitsÚ	lm_logitsÚaatypeÚatom14_atom_existsÚresidx_atom14_to_atom37Úresidx_atom37_to_atom14Úatom37_atom_existsÚresidue_indexÚ	lddt_headÚplddtÚ
ptm_logitsÚptmÚaligned_confidence_probsÚpredicted_aligned_errorÚmax_predicted_aligned_error)ÚtorchÚFloatTensor)ÚformatÚ__classdict__s   "€Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/esm/modeling_esmfold.pyÚ__annotate__Ú'EsmForProteinFoldingOutput.__annotate__6   s[  ø‡ ‚ ñt ×Ñ Õ$Ñ+ñu ñv ×'Ñ'¨$Õ.Ñ5ñw ñx ×*Ñ*¨TÕ1Ñ8ñy ñz ×Ñ Õ$Ñ+ñ{ ñ| × Ñ  4Õ'Ñ.ñ} ñ~ ×Ñ Õ$Ñ+ñ ñ@ 
×	Ñ	˜TÕ	!Ñ(ñA ñB 
×	Ñ	˜TÕ	!Ñ(ñC ñD ×'Ñ'¨$Õ.Ñ5ñE ñF × Ñ  4Õ'Ñ.ñG ñH ×Ñ Õ$Ñ+ñI ñJ ×)Ñ)¨DÕ0Ñ7ñK ñL #×.Ñ.°Õ5Ñ<ñM ñN #×.Ñ.°Õ5Ñ<ñO ñP ×)Ñ)¨DÕ0Ñ7ñQ ñR ×$Ñ$ tÕ+Ñ2ñS ñT × Ñ  4Õ'Ñ.ñU ñV ×Ñ˜tÕ#Ñ*ñW ñX ×!Ñ! DÕ(Ñ/ñY ñZ 
×	Ñ	˜TÕ	!Ñ(ñ[ ñ\ $×/Ñ/°$Õ6Ñ=ñ] ñ^ #×.Ñ.°Õ5Ñ<ñ_ ñ` "'×!2Ñ!2°TÕ!9Ñ@òa ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r<   s   @r=   r   r   6   s’   ø‡ € ñ1ðf (,€FØ15ÐØ48ÐØ'+€FØ*.€IØ'+€FØ$(€CØ$(€CØ15ÐØ*.€IØ'+€FØ37ÐØ8<ÐØ8<ÐØ37ÐØ.2€MØ*.€IØ&*€EØ+/€JØ$(€CØ9=ÐØ8<ÐØ<@Ð÷a ƒ r@   r   c                 ó˜   € \         P                  ! V 4      pV\         P                  8H  pT;'       d    \         P                  ! V 4      pV# ©N)r9   Úget_autocast_dtypeÚfloat16Úis_autocast_enabled)Údevice_typeÚautocast_dtypeÚfp16_enableds   &  r=   Úis_fp16_enabledrS   ‰   s>   € ä×-Ò-¨kÓ:€NØ!¤U§]¡]Ñ2€LØ×JÐJ¤E×$=Ò$=¸kÓ$J€LàÐr@   c                  óˆ   € \        4       '       d   R #  ^ RIp V P                  P                  4       #   \         d     R # i ; i©FN)r	   Ú	deepspeedÚutilsÚis_initializedÚ	Exception)rV   s    r=   Úis_deepspeed_initializedrZ   ’   s=   € Ü×ÒÙð	Ûð —?‘?×1Ñ1Ó3Ð3øÜô 	Úð	ús   ”2 ²AÁ Ac                óz   € V ^8„  d   QhR\         \        P                  ,          R\        R\        P                  /# )r!   ÚsamplesÚpad_vÚreturn)Úlistr9   ÚTensorÚfloat)r;   s   "r=   r>   r>   Ÿ   s.   € ÷ ñ ¤4¬¯©Õ#5ð ¼eð ÌEÏLÉLñ r@   c                óZ  € \        V 4      ^ 8X  d   \        P                  ! 4       # \        V  Uu0 uF  q"P                  4       kK  	  up4      ^8w  d,   \	        RV  Uu. uF  q"P                  4       NK  	  up 24      h\        V  Uu0 uF  q"P                  kK  	  up4      w  p\        V  Uu. uF  q"P                  NK  	  up!   Uu. uF  p\        V4      NK  	  pp\        P                  ! \        V 4      .VO5RV ^ ,          P                  RV/ pVP                  V4       \        \        V 4      4       FU  pWg,          pW,          p	Y˜\
        ;QJ d    . R V	P                   4       F  NK  	  5M! R V	P                   4       4      &   KW  	  V# u upi u upi u upi u upi u upi )zÝ
Takes a list of tensors with the following dimensions:
    [(d_11, ..., d_1K),
     (d_21, ..., d_2K), ..., (d_N1, ..., d_NK)]
and stack + pads them into a single tensor of:
(N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK})
z Samples has varying dimensions: ÚdtypeÚdevicec              3   ó:   "  € T F  p\        ^ V4      x € K  	  R# 5i)r   N©Úslice)Ú.0Úks   & r=   Ú	<genexpr>Ú(collate_dense_tensors.<locals>.<genexpr>²   s   é € Ð4©G q”u˜Q —{�{«Gùs   ‚)Úlenr9   r`   ÚdimÚRuntimeErrorÚtuplerd   ÚzipÚshapeÚmaxÚemptyrc   Úfill_Úrange)
r\   r]   Úxrd   ÚlstÚ	max_shapeÚresultÚiÚresult_iÚts
   &&        r=   Úcollate_dense_tensorsr}   Ÿ   sM  € ô ˆ7ƒ|�qÔÜ�|Š|‹~ÐÜ
™WÓ%™W˜�E‰EŽG™WÑ%Ó&¨!Ô+ÜÐ=ÑPWÓ>XÑPWÈ1¿u¹u¾wÑPWÑ>XÐ=YÐZÓ[Ð[Ü©Ó1© A—x”x©Ñ1Ó2�I€VÜ%(¹GÓ*D¹G°q¯7¬7¹GÑ*DÒ%EÓFÑ%E˜c”�S–Ñ%E€IÐFÜ�[Š[œ˜W›ÐY¨	ÒY¸À½×9IÑ9IÐYÐRXÑY€FØ
‡L�L�ÔÜ”3�w“<Ö ˆØ•9ˆØ�JˆØ89—”Ñ4¨A¯GªGÓ4—‘Ñ4¨A¯GªGÓ4Ó4Ó5ñ !ð €Mùò &ùÚ>XùÚ1ùÚ*DùÒFs   ¯FÁF
ÂFÂ-F#ÃF(c                óD   € V ^8„  d   QhR\         P                  R\        /# )r!   r|   Úno_dims©r9   r`   Úint)r;   s   "r=   r>   r>   ¶   s   € ÷ 1ñ 1œ%Ÿ,™,ð 1´ñ 1r@   c                 óN   € V P                  V P                  R V)  R,           4      # )N©éÿÿÿÿ)Úreshaperq   )r|   r   s   &&r=   Úflatten_final_dimsr†   ¶   s#   € Ø�9‰9�Q—W‘W˜Y˜w˜hÐ'¨%Õ/Ó0Ð0r@   c                óZ   € V ^8„  d   QhR\         P                  R\        \        ,          /# )r!   ÚtensorÚinds)r9   r`   r_   r�   )r;   s   "r=   r>   r>   º   s&   € ÷ Gñ GœuŸ|™|ð G´4¼µ9ñ Gr@   c           	      óÞ   € R\        V4      ,          p\        \        \        V P                  RV 4      4      4      pT P	                  Y1 Uu. uF  qBV,           NK  	  up,           4      # u upi )é   Nr„   )rl   r_   ru   rq   Úpermute)rˆ   r‰   Ú
zero_indexÚ
first_indsrz   s   &&   r=   Úpermute_final_dimsr�   º   sX   € Ø”c˜$“i•€JÜ”eœC §¡¨[¨jÐ 9Ó:Ó;Ó<€JØ�>‰>˜*ÀÓ'EÁ¸1°Q¯¨ÁÑ'EÕEÓFÐFùÒ'Es   ÁA*
c                 óä   € V^ ,          p/ pVP                  4        FK  w  rEV Uu. uF  qfV,          NK  	  pp\        V\        4      '       d   \        W4      W4&   KA  V ! V4      W4&   KM  	  V# u upi ©r   )ÚitemsÚ
isinstanceÚdictÚdict_multimap)ÚfnÚdictsÚfirstÚnew_dictri   ÚvÚdÚall_vs   &&      r=   r•   r•   À   sj   € Ø�!�H€EØ€HØ—‘–‰ˆÙ$Ó%™u˜!�1—�™uˆÐ%Ü�aœ×ÒÜ'¨Ó2ˆH‹Ká˜U›)ˆH‹Kñ ð €Oùò &s   ¥A-c                   óF   a a€ ] tR t^Ít oRtRV3R lV 3R llltRtVtV ;t# )ÚEsmFoldLinearz²
A Linear layer with built-in nonstandard initializations. Called just like torch.nn.Linear.

Implements the initializers in 1.11.4, plus some additional ones found in the code.
c                óˆ   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[S[P
                  S[P
                  .R3,          R,          /# )r!   Úin_dimÚout_dimÚbiasÚinitÚinit_fnN)r�   ÚboolÚstrr   r9   r`   )r;   r<   s   "€r=   r>   ÚEsmFoldLinear.__annotate__Ô   sY   ø€ ÷ $5ñ $5áð$5ñ ð$5ñ ð	$5ñ
 ð$5ñ ™5Ÿ<™<©¯©Ð6¸Ð<Õ=ÀÕDñ$5r@   c                ó  <€ \         SV `  WVR7       V'       dB   \        P                  ! 4       ;_uu_ 4        V P                  P                  ^ 4       RRR4       W@n        WPn        VR9  d   \        R4      hR#   + '       g   i     L0; i)aÍ  
Args:
    in_dim:
        The final dimension of inputs to the layer
    out_dim:
        The final dimension of layer outputs
    bias:
        Whether to learn an additive bias. True by default
    init:
        The initializer to use. Choose from:

        "default": LeCun fan-in truncated normal initialization "relu": He initialization w/ truncated normal
        distribution "glorot": Fan-average Glorot uniform initialization "gating": Weights=0, Bias=1 "normal":
        Normal initialization with std=1/sqrt(fan_in) "final": Weights=0, Bias=0

        Overridden by init_fn if the latter is not None.
    init_fn:
        A custom initializer taking weight and bias as inputs. Overrides init if not None.
©r¢   NzInvalid init string.)ÚdefaultÚreluÚglorotÚgatingÚnormalÚfinal)	ÚsuperÚ__init__r9   Úno_gradr¢   rt   r£   r¤   Ú
ValueError)Úselfr    r¡   r¢   r£   r¤   Ú	__class__s   &&&&&&€r=   r±   ÚEsmFoldLinear.__init__Ô   sg   ø€ ô6 	‰Ñ˜¨tÐÔ4çÜ—’—•Ø—	‘	—‘ Ô"÷ !àŒ	ØŒàÐQÔQÜÐ3Ó4Ð4ñ R÷ !—ús   ¸A<Á<B	)r£   r¤   )Trª   N)	rB   rC   rD   rE   rF   r±   rH   rI   Ú__classcell__©rµ   r<   s   @@r=   rž   rž   Í   s   ù‡ € ñ÷$5÷ $5õ $5r@   rž   c                   ó<   a a€ ] tR t^ût oRV 3R lltR tRtVtV ;t# )ÚEsmFoldLayerNormc                óü   <€ \         SV `  4        V3V n        W n        \        P
                  ! \        P                  ! V4      4      V n        \        P
                  ! \        P                  ! V4      4      V n
        R # rL   )r°   r±   Úc_inÚepsÚnnÚ	Parameterr9   ÚonesÚweightÚzerosr¢   )r´   r¼   r½   rµ   s   &&&€r=   r±   ÚEsmFoldLayerNorm.__init__ü   sN   ø€ Ü‰ÑÔà�GˆŒ	ØŒä—l’l¤5§:¢:¨dÓ#3Ó4ˆŒÜ—L’L¤§¢¨TÓ!2Ó3ˆŽ	r@   c                ó6  € VP                   pV\        P                  J d›   \        4       '       g‹   \	        R RR7      ;_uu_ 4        \
        P                  P                  WP                  V P                  P                  VR7      V P                  P                  VR7      V P                  4      pRRR4       V# \
        P                  P                  WP                  V P                  V P                  V P                  4      pV#   + '       g   i     X# ; i)ÚcudaF©rP   Úenabled©rc   N)rc   r9   Úbfloat16rZ   r   r¾   Ú
functionalÚ
layer_normr¼   rÁ   Útor¢   r½   )r´   rv   r›   Úouts   &&  r=   ÚforwardÚEsmFoldLayerNorm.forward  sÊ   € Ø�G‰GˆØ”—‘ÓÔ'?×'AÒ'AÜ¨F¸E×BÖBÜ—m‘m×.Ñ.¨q·)±)¸T¿[¹[¿^¹^ÐRS¸^Ó=TÐVZ×V_ÑV_×VbÑVbÐijÐVbÓVkÐmq×muÑmuÓv�÷ Cð
 ˆ
ô —-‘-×*Ñ*¨1¯i©i¸¿¹ÀdÇiÁiÐQU×QYÑQYÓZˆCàˆ
÷ CÖBð
 ˆ
ús   ÁA+DÄD	)r¢   r¼   r½   rÁ   )gñhãˆµøä>©	rB   rC   rD   rE   r±   rÎ   rH   rI   r·   r¸   s   @@r=   rº   rº   û   s   ù‡ € ÷4÷ò r@   rº   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r!   r|   rm   r^   r€   )r;   s   "r=   r>   r>     s)   € ÷ ñ ”u—|‘|ð ¬#ð ´u·|±|ñ r@   c                óv  € V P                   pV\        P                  J d[   \        4       '       gK   \	        RRR7      ;_uu_ 4        \        P
                  P                  P                  WR7      pRRR4       V# \        P
                  P                  P                  WR7      pV#   + '       g   i     X# ; i)zS
Softmax, but without automatic casting to fp32 when the input is of type bfloat16
rÅ   FrÆ   ©rm   N)rc   r9   rÉ   rZ   r   r¾   rÊ   Úsoftmax)r|   rm   r›   Úss   &&  r=   Úsoftmax_no_castrÖ     sŒ   € ð
 	
�‰€AØŒE�N‰NÓÔ#;×#=Ò#=Ü¨¸×>Ö>Ü—‘×#Ñ#×+Ñ+¨AÐ+Ó7ˆA÷ ?ð
 €Hô �H‰H×Ñ×'Ñ'¨Ð'Ó3ˆà€H÷ ?Ö>ð
 €Hús   Á+B'Â'B8	c                   ó€   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltV3R lR ltRV3R	 lR
 lltRt	Vt
V ;t# )ÚEsmFoldAttentioni  zm
Standard multi-head attention using AlphaFold's default layer initialization. Allows multiple bias vectors.
c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[/# )r!   Úc_qÚc_kÚc_vÚc_hiddenÚno_headsr­   )r�   r¥   )r;   r<   s   "€r=   r>   ÚEsmFoldAttention.__annotate__$  sG   ø€ ÷ -$ñ -$áð-$ñ ð-$ñ ð	-$ñ
 ð-$ñ ð-$ñ ñ-$r@   c                ó  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        W`n        \        V P                  V P
                  V P                  ,          RRR7      V n	        \        V P                  V P
                  V P                  ,          RRR7      V n
        \        V P                  V P
                  V P                  ,          RRR7      V n        \        V P
                  V P                  ,          V P                  RR7      V n        RV n        V P                  '       d9   \        V P                  V P
                  V P                  ,          RR7      V n        \        P                  ! 4       V n        R# )a:  
Args:
    c_q:
        Input dimension of query data
    c_k:
        Input dimension of key data
    c_v:
        Input dimension of value data
    c_hidden:
        Per-head hidden dimension
    no_heads:
        Number of attention heads
    gating:
        Whether the output should be gated using query data
Fr¬   ©r¢   r£   r¯   ©r£   Nr­   )r°   r±   rÚ   rÛ   rÜ   rÝ   rÞ   r­   rž   Úlinear_qÚlinear_kÚlinear_vÚlinear_oÚlinear_gr¾   ÚSigmoidÚsigmoid)r´   rÚ   rÛ   rÜ   rÝ   rÞ   r­   rµ   s   &&&&&&&€r=   r±   ÚEsmFoldAttention.__init__$  s   ø€ ô0 	‰ÑÔàŒØŒØŒØ ŒØ ŒØŒô
 & d§h¡h°·±ÀÇÁÕ0MÐTYÐ`hÔiˆŒÜ% d§h¡h°·±ÀÇÁÕ0MÐTYÐ`hÔiˆŒÜ% d§h¡h°·±ÀÇÁÕ0MÐTYÐ`hÔiˆŒÜ% d§m¡m°d·m±mÕ&CÀTÇXÁXÐT[Ô\ˆŒàˆŒØ�;�;ˆ;Ü)¨$¯(©(°D·M±MÀDÇMÁMÕ4QÐX`ÔaˆDŒMä—z’z“|ˆŽr@   c          	      óª   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  S[ P                  3,          /# )r!   Úq_xÚkv_xr^   ©r9   r`   ro   )r;   r<   s   "€r=   r>   rß   S  sH   ø€ ÷ ñ ™UŸ\™\ð ±·±ð Á%ÉÏÉÑV[×VbÑVbÑdi×dpÑdpÐHpÕBqñ r@   c                óP  € V P                  V4      pV P                  V4      pV P                  V4      pVP                  VP                  R R V P
                  R3,           4      pVP                  VP                  R R V P
                  R3,           4      pVP                  VP                  R R V P
                  R3,           4      pVP                  RR4      pVP                  RR4      pVP                  RR4      pV\        P                  ! V P                  4      ,          pW4V3# )Nr„   éþÿÿÿéýÿÿÿ)
rã   rä   rå   Úviewrq   rÞ   Ú	transposeÚmathÚsqrtrÝ   )r´   rì   rí   Úqri   rš   s   &&&   r=   Ú	_prep_qkvÚEsmFoldAttention._prep_qkvS  sò   € à�M‰M˜#ÓˆØ�M‰M˜$ÓˆØ�M‰M˜$Óˆð �F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Õ5Ó6ˆØ�F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Õ5Ó6ˆØ�F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Õ5Ó6ˆð �K‰K˜˜BÓˆØ�K‰K˜˜BÓˆØ�K‰K˜˜BÓˆà	ŒT�YŠY�t—}‘}Ó%Õ%ˆà�Qˆwˆr@   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r!   Úorì   r^   ©r9   r`   )r;   r<   s   "€r=   r>   rß   g  s.   ø€ ÷ ñ ™%Ÿ,™,ð ©U¯\©\ð ¹e¿l¹lñ r@   c                ó  € V P                   eZ   V P                  V P                  V4      4      pVP                  VP                  R R V P                  R3,           4      pW,          p\        V^4      pV P                  V4      pV# )Nr„   )rç   ré   rò   rq   rÞ   r†   ræ   )r´   rú   rì   Úgs   &&& r=   Ú_wrap_upÚEsmFoldAttention._wrap_upg  su   € Ø�=‰=Ò$Ø—‘˜TŸ]™]¨3Ó/Ó0ˆAð —‘�q—w‘w˜s �| t§}¡}°bÐ&9Õ9Ó:ˆAØ•ˆAô ˜q !Ó$ˆð �M‰M˜!Óˆàˆr@   c                óæ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          R,          RS[RS[RS[RS[R	S[R
S[ P                  R,          RS[ P                  /
# )r!   rì   rí   ÚbiasesNÚuse_memory_efficient_kernelÚuse_lmaÚlma_q_chunk_sizeÚlma_kv_chunk_sizeÚ	use_flashÚ
flash_maskr^   )r9   r`   r_   r¥   r�   )r;   r<   s   "€r=   r>   rß   w  s’   ø€ ÷ =ñ =á�\‰\ð=ñ �l‰lð=ñ ‘U—\‘\Õ" TÕ)ð	=ñ
 &*ð=ñ ð=ñ ð=ñ ð=ñ ð=ñ —L‘L 4Õ'ð=ñ 
�‰ñ=r@   c
                óÌ  € V'       d   Ve   Vf   \        R4      hV'       d   Ve   \        R4      hWEV.p
\        V
4      ^8”  d   \        R4      hVf   . pV P                  W4      w  r¼p\        VR4      p\        P
                  ! W¼4      pV F  pWï,          pK  	  \        VR4      p\        P
                  ! Wí4      pVP                  RR4      pV P                  Wá4      pV# )aÛ  
Args:
    q_x:
        [*, Q, C_q] query data
    kv_x:
        [*, K, C_k] key data
    biases:
        List of biases that broadcast to [*, H, Q, K]
    use_memory_efficient_kernel:
        Whether to use a custom memory-efficient attention kernel. This should be the default choice for most.
        If none of the "use_<...>" flags are True, a stock PyTorch implementation is used instead
    use_lma:
        Whether to use low-memory attention (Staats & Rabe 2021). If none of the "use_<...>" flags are True, a
        stock PyTorch implementation is used instead
    lma_q_chunk_size:
        Query chunk size (for LMA)
    lma_kv_chunk_size:
        Key/Value chunk size (for LMA)
Returns
    [*, Q, C_q] attention update
zPIf use_lma is specified, lma_q_chunk_size and lma_kv_chunk_size must be providedzSuse_flash is incompatible with the bias option. For masking, use flash_mask insteadz2Choose at most one alternative attention algorithm)r‹   r   r„   rð   rñ   )	r³   Úsumr÷   r�   r9   ÚmatmulrÖ   ró   rþ   )r´   rì   rí   r  r  r  r  r  r  r  Úattn_optionsÚqueryÚkeyÚvalueÚoutputÚbs   &&&&&&&&&&      r=   rÎ   ÚEsmFoldAttention.forwardw  sä   € ÷B Ð(Ò0Ð4EÒ4MÜÐoÓpÐpç˜Ò+ÜÐrÓsÐsà3¸iÐHˆÜˆ|Ó˜qÔ ÜÐQÓRÐRàŠ>ØˆFð !ŸN™N¨3Ó5Ñˆ�EÜ   fÓ-ˆô —’˜eÓ)ˆÛˆAØ�KŠFñ ä  ¨Ó,ˆô —’˜fÓ,ˆØ×!Ñ! " bÓ)ˆØ—‘˜vÓ+ˆàˆr@   )rÝ   rÛ   rÚ   rÜ   r­   rç   rä   ræ   rã   rå   rÞ   ré   ©T)NFFi   i   FN)rB   rC   rD   rE   rF   r±   r÷   rþ   rÎ   rH   rI   r·   r¸   s   @@r=   rØ   rØ     s8   ù‡ € ñ÷-$õ -$÷^ð ÷(ð ÷ =÷ =ò =r@   rØ   c                   ó”   a a€ ] tR tRt oRV 3R llt]P                  P                  R	V3R lR ll4       tR
V3R lR llt	Rt
VtV ;t# )ÚEsmFoldTriangleAttentioni·  c                óZ  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        \        V P                  4      V n        \        WP                  RRR7      V n
        \        V P                  V P                  V P                  V P                  V P                  4      V n        R# )z§
Args:
    c_in:
        Input channel dimension
    c_hidden:
        Overall hidden channel dimension (not per-head)
    no_heads:
        Number of attention heads
Fr®   rá   N)r°   r±   r¼   rÝ   rÞ   ÚstartingÚinfr   rË   rž   ÚlinearrØ   Úmha)r´   r¼   rÝ   rÞ   r  r  rµ   s   &&&&&&€r=   r±   Ú!EsmFoldTriangleAttention.__init__¸  sx   ø€ ô 	‰ÑÔàŒ	Ø ŒØ ŒØ ŒØŒä# D§I¡IÓ.ˆŒä# D¯-©-¸eÈ(ÔSˆŒä# D§I¡I¨t¯y©y¸$¿)¹)ÀTÇ]Á]ÐTX×TaÑTaÓbˆŽr@   c                ó�   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[RS[RS[RS[RS[ P                  /# )r!   rv   r  Ú
chunk_sizer  r  Úinplace_safer^   )r9   r`   r_   r�   r¥   )r;   r<   s   "€r=   r>   Ú%EsmFoldTriangleAttention.__annotate__Ñ  sb   ø€ ÷ 
ñ 
á�<‰<ð
ñ ‘U—\‘\Õ"ð
ñ ð	
ñ
 &*ð
ñ ð
ñ ð
ñ 
�‰ñ
r@   c           
     ó¦   € RVRVRV/p\        \        V P                  WER7      TT\        VP                  RR 4      V'       d   VR7      # RR7      # )ztriangle! triangle!rì   rí   r  )r  r  N)r  Úno_batch_dimsÚ_outrð   )r   r   r  rl   rq   )r´   rv   r  r  r  r  r  Ú
mha_inputss   &&&&&&& r=   Ú_chunkÚEsmFoldTriangleAttention._chunkÐ  sd   € ð �1Ø�AØ�fð
ˆ
ô Ü�D—H‘HÐ:UÔgØØ!Ü˜aŸg™g c r˜lÓ+ß"�ô
ð 	
ð
 )-ô
ð 	
r@   c                óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[RS[RS[RS[ P                  /# )	r!   rv   ÚmaskNr  r  r  r  r^   ©r9   r`   r�   r¥   )r;   r<   s   "€r=   r>   r  é  sf   ø€ ÷ 9ñ 9á�<‰<ð9ñ �l‰l˜TÕ!ð9ñ ˜$•Jð	9ñ
 &*ð9ñ ð9ñ ð9ñ 
�‰ñ9r@   c           	     ó   € Vf   VP                  VP                  RR 4      pV P                  '       g%   VP                  RR4      pVP                  RR4      pV P	                  V4      pV P
                  V^,
          ,          R,          p\        V P                  V4      R4      pVP                  R	4      pWx.p	Ve   V P                  VV	VVVVR7      pMV P                  WW”VR7      pV P                  '       g   VP                  RR4      pV# )
z}
Args:
    x:
        [*, I, J, C_in] input tensor (e.g. the pair representation)
Returns:
    [*, I, J, C_in] output tensor
N)r  r  r  )rì   rí   r  r  r  r„   rð   rñ   ).ºNNNNNr)  ©r!   r   r‹   éüÿÿÿ)Únew_onesrq   r  ró   rË   r  r�   r  Ú	unsqueezer#  r  )
r´   rv   r&  r  r  r  r  Ú	mask_biasÚtriangle_biasr  s
   &&&&&&&   r=   rÎ   Ú EsmFoldTriangleAttention.forwardé  s  € ð  Š<à—:‘:Ø—‘˜˜�óˆDð �}�}ˆ}Ø—‘˜B Ó#ˆAØ—>‘> " bÓ)ˆDð �O‰O˜AÓˆð —X‘X ¨¥Õ*Ð,AÕBˆ	ô +¨4¯;©;°q«>¸9ÓEˆð &×/Ñ/°Ó3ˆàÐ+ˆàÒ!Ø—‘ØØØØ,GØØ)ð ó ‰Að —‘Ø fÐovð ó ˆAð �}�}ˆ}Ø—‘˜B Ó#ˆAàˆr@   )rÝ   r¼   r  rË   r  r  rÞ   r  )Tç    eÍÍA)FFF)NNFFF)rB   rC   rD   rE   r±   r9   ÚjitÚignorer#  rÎ   rH   rI   r·   r¸   s   @@r=   r  r  ·  s=   ù‡ € ÷cð0 ‡Y�Y×Ñ÷
ñ 
ó ð
÷09÷ 9ò 9r@   r  c                   ó|   a a€ ] tR tRt oRtRV 3R lltRV3R lR lltRV3R lR lltRV3R lR	 lltR
t	Vt
V ;t# )Ú#EsmFoldTriangleMultiplicativeUpdatei%  z"
Implements Algorithms 11 and 12.
c                óŒ  <€ \         SV `  4        VP                  pW n        \	        W34      V n        \	        W3R R7      V n        \	        W34      V n        \	        W3R R7      V n        \	        W3R R7      V n	        \	        W3RR7      V n
        \        V4      V n        \        V4      V n        \        P                  ! 4       V n        R# )r­   râ   r¯   N)r°   r±   Úpairwise_state_dimÚ	_outgoingrž   Ú
linear_a_pÚ
linear_a_gÚ
linear_b_pÚ
linear_b_grç   Úlinear_zr   Úlayer_norm_inÚlayer_norm_outr¾   rè   ré   )r´   Úconfigr8  rÝ   rµ   s   &&& €r=   r±   Ú,EsmFoldTriangleMultiplicativeUpdate.__init__*  s•   ø€ Ü‰ÑÔØ×,Ñ,ˆØ"Œä'¨Ó;ˆŒÜ'¨ÀÔJˆŒÜ'¨Ó;ˆŒÜ'¨ÀÔJˆŒÜ% h¸xÔHˆŒÜ% h¸wÔGˆŒä& xÓ0ˆÔÜ'¨Ó1ˆÔä—z’z“|ˆŽr@   c                ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[R,          RS[ P                  /# )r!   Úar  Ú_inplace_chunk_sizeNr^   r€   )r;   r<   s   "€r=   r>   Ú0EsmFoldTriangleMultiplicativeUpdate.__annotate__;  s=   ø€ ÷ 0ñ 0Ù—‘ð0Ù"'§,¡,ð0ÙEHÈ4ÅZð0á	�‰ñ0r@   c                óÄ  € V P                   '       d   \        VR4      p\        VR4      pM\        VR4      p\        VR4      pVey   \        ^ VP                  R,          V4       FS  pVRWDV,           1RR3,          pVRWDV,           1RR3,          p\        P
                  ! VV4      VRWDV,           1RR3&   KU  	  TpM\        P
                  ! W4      p\        VR4      # )r!   .r)  r*  )r!   r‹   r   rñ   ©r‹   r!   r   )r8  r�   ru   rq   r9   r
  )r´   rC  r  rD  rz   Úa_chunkÚb_chunkÚps   &&&&    r=   Ú_combine_projectionsÚ8EsmFoldTriangleMultiplicativeUpdate._combine_projections;  så   € ð �>�>ˆ>Ü" 1 iÓ0ˆAÜ" 1 iÓ0‰Aä" 1 iÓ0ˆAÜ" 1 iÓ0ˆAàÒ*ä˜1˜aŸg™g b�kÐ+>Ö?�Ø˜C Ð)<Õ%<Ð!<¸aÀÐBÕC�Ø˜C Ð)<Õ%<Ð!<¸aÀÐBÕC�Ü<A¿LºLØØó=��#�qÐ2Õ2Ð2°A°qÐ8Ó9ñ @ð ‰Aä—’˜QÓ"ˆAä! ! YÓ/Ð/r@   c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[R,          RS[/# )r!   Úzr&  NÚinplace_chunk_sizeÚwith_addr'  )r;   r<   s   "€r=   r>   rE  U  sJ   ø€ ÷ añ aá�<‰<ðañ �l‰l˜TÕ!ðañ   $�Jð	añ
 ñar@   c                ó¦  a aa#a$a%a&a'a(a)€ Vf   VP                  VP                  RR 4      pVP                  R4      pRV 3R llo$RV$VV 3R llpV! WRRR7      pSEeæ   VP                  R,          o'S'^,          S'^,          ,           o&Ro(Ro#S P                  '       d   S(MS#pR o%V%3R lo)V#V%V&V'V(V)3R lp\	        VP                  4      p	S&V	S#&   VP                  V	4      p
S%! V
4      p\        ^ S&4      VS#&   V
P                  W,          4       R	p\	        \        ^ S&S4      4      p\        WÝR
,          S&.,           4       UUu. uF  w  rïWþ,
          NK  	  ppp\	        \        S&S'S4      4      pV Uu. uF  pSNK  	  pp\        VV,           VV,           4      pV EFŸ  w  ppV'       g   VS&8¼  d   V! W¡4      p
RpS)! VVVV,           V4      pS)! VVVV,           V4      pVP                  4       pVS#8X  d   S)! VVVV,           S#4      pMPV'       g.   S%! V4      p\        ^ S&4      VS#&   S)! V
VVV,           S(4      VV&   MVS&,
          pS)! V
VVV,           S(4      pV! VVR	R	R7      p?\        P                  ! VV4      p\        VR4      pS P                  V4      pS P                  V4      pS)! VVVV,           S#4      pS P!                  S P#                  V4      4      pVP%                  4        ?VV,          pS%! V4      p\        VVV,           4      VS#&   V'       d   VV;;,          V,          uu&   EKš  VVV&   EK¢  	  V# V! WR	R	4      p \        P                  ! VV 4      p!S P                  V!4      p!S P                  V!4      p!S P!                  V4      p"V"P%                  4        V!V",          p!V'       d   VV!,          pV# T!pV# u uppi u upi )aa  
Args:
    z:
        A [*, N, N, C_z] pair representation
    mask:
        A [*, N, N] pair mask
    inplace_chunk_size:
        Size of chunks used in the main computation. Increase to trade memory for speed.
    with_add:
        If True, z is overwritten with (z + update). Otherwise, it is overwritten with (update).
Returns:
    A reference to the overwritten z

More memory-efficient, inference-only version of the forward function. Uses in-place operations, fusion of the
addition that happens after this module in the Evoformer, a smidge of recomputation, and a cache of overwritten
values to lower peak memory consumption of this module from 5x the size of the input tensor z to 2.5x its size.
Useful for inference on extremely long sequences.

It works as follows. We will make reference to variables used in the default forward implementation below.
Naively, triangle multiplication attention requires the manifestation of 5 tensors the size of z: 1) z, the
"square" input tensor, 2) a, the first projection of z, 3) b, the second projection of b, 4) g, a z-sized mask,
and 5) a z-sized tensor for intermediate computations. For large N, this is prohibitively expensive; for
N=4000, for example, z is more than 8GB alone. To avoid this problem, we compute b, g, and all intermediate
tensors in small chunks, noting that the chunks required to compute a chunk of the output depend only on the
tensor a and corresponding vertical and horizontal chunks of z. This suggests an algorithm that loops over
pairs of chunks of z: hereafter "columns" and "rows" of z, even though each "column" and "row" in fact contains
inplace_chunk_size contiguous true columns and rows of z. Writing output chunks to a new tensor would bring
total memory consumption down to 3x the size of z. However, more memory can be saved by writing output chunks
directly to z in-place. WLOG, we choose to write output chunks vertically, overwriting the ith "column" of z at
the end of the ith iteration of the main loop. Despite this overwriting, the ith column is always one column
ahead of previously overwritten columns and can be recovered directly from z. After the first iteration,
however, the ith row of z is always at least partially overwritten. For this reason, we introduce the z-cache,
a tensor one-half the size of z. The z-cache initially contains the left half (2nd and 3rd quadrants) of z. For
0 < i < N/2, the missing left part of the ith row of z is recovered from this cache at the beginning of the ith
iteration. Once i exceeds n/2, the cache is "reoriented" to encompass the 3rd and 4th quadrants of z instead.
Though the 3rd quadrant of the original z is entirely overwritten at this point, it can be recovered from the
z-cache itself. Thereafter, the ith row of z can be recovered in its entirety from the reoriented z-cache.
After the final iteration, z has been completely overwritten and contains the triangular multiplicative update.
If with_add is True, it instead contains the sum of z and the triangular multiplicative update. In either case,
peak memory consumption is just 2.5x the size of z, disregarding memory used for chunks and other small
variables.
NTc                 ó  <€ V'       d   SP                   pSP                  pMSP                  pSP                  pSP	                  V 4      p V! V 4      pVP                  4        WT! V 4      ,          pWQ,          p\        VR4      pV# )r!   r*  )r:  r9  r<  r;  r>  Úsigmoid_r�   )Úpairr&  rC  rç   Úlinear_prJ  r´   s   &&&   €r=   Úcompute_projection_helperÚYEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projection_helper‹  sr   ø€ ßØŸ?™?�ØŸ?™?‘àŸ?™?�ØŸ?™?�à×%Ñ% dÓ+ˆDÙ˜“ˆAØ�J‰JŒLØ�˜$“ÕˆAØ�IˆAÜ" 1 iÓ0ˆAØˆHr@   c           	      óÀ  <€ SP                   V,          pV'       g&   S! WV4      pV'       d   VP                  RR4      pV# V'       d   SP                  MSP                  pVP                  P
                  R,          pV P
                  RR V3,           V P
                  RR ,           pV P                  V4      p\        ^ V P
                  R,          S4       F�  p	V RW™S,           1RR3,          p
S! V RW™S,           1RR3,          VRW™S,           1RR3,          V4      p
V'       d"   V
P                  RR4      p
W¥RW™S,           13&   MW¥RW™S,           1R3&   ?
Kƒ  	  V# )r‹   N.r)  r„   rð   rñ   )r8  ró   r:  r<  r¢   rq   Ú	new_zerosru   )rT  r&  rC  ÚchunkedÚneed_transposerJ  rç   ÚcÚ	out_shaperz   Ú
pair_chunkrV  rO  r´   s   &&&&       €€€r=   Úcompute_projectionÚREsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projection›  s^  ø€ Ø!Ÿ^™^¨aÕ/ˆNßÙ-¨d¸!Ó<�ß!ØŸ™ B¨Ó+�Að. ˆH÷' /0˜4Ÿ?š?°T·_±_�Ø—M‘M×'Ñ'¨Õ+�Ø ŸJ™J s¨˜O¨q¨dÕ2°T·Z±ZÀÀ2Ð5FÕF�	Ø—N‘N 9Ó-�Ü˜q $§*¡*¨R¥.Ð2DÖE�AØ!% c¨1Ð3EÕ/EÐ+EÀqÈ!Ð&KÕ!L�JÙ!:Ø˜S !Ð*<Õ&<Ð"<¸aÀÐBÕCØ˜S !Ð*<Õ&<Ð"<¸aÀÐBÕCØó"�J÷
 &Ø%/×%9Ñ%9¸"¸bÓ%A˜
Ø=G˜#˜qÐ'9Õ#9Ð9Ð9Ò:à@J˜#˜qÐ'9Õ#9Ð9¸1Ð<Ñ=â"ñ Fð ˆHr@   )rZ  c                 óV   € V P                    Uu. uF  p\        R 4      NK  	  up# u upi rL   )rq   rg   )r|   Ú_s   & r=   Úempty_slicerÚLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.empty_slicerÅ  s"   € Ø-.¯WªWÓ5©W¨œ˜dž©WÑ5Ð5ùÒ5s   �&c                 ó>   <€ S! V 4      p\        W4      WC&   W,          # rL   rf   )r|   ÚstartÚendrm   rÕ   rc  s   &&&& €r=   Úslice_tensorÚLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.slice_tensorÈ  s   ø€ á  “O�Ü˜uÓ*�‘Ø•t�r@   c                 ó  <€ S! V SR S
4      pV P                  S
S4      p V RR S	^,          1RR3,          p S! V 4      p\        ^ S4      VS&   W V&   S! VSR S
4      pS! VSR S4      pS! V 4      p\        SR 4      VS&   W@V&   V # )N.r)  )ró   rg   )Úz_cacherN  Ú
quadrant_3Úfirst_half_slicerÚ
quadrant_4Úquadrant_3_slicerÚcol_dimrc  Úhalf_nÚnÚrow_dimrh  s   &&    €€€€€€r=   Úflip_z_cache_ÚMEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.flip_z_cache_Î  s»   ø€ ñ *¨'°6¸4ÀÓI�
Ø!×+Ñ+¨G°WÓ=�ð " # z¨!¨q­& z°1°aÐ"7Õ8�ñ %1°Ó$9Ð!Ü-2°1°fÓ-=Ð! 'Ñ*Ø-7Ð)Ñ*ñ *¨!¨V°T¸7ÓC�
Ù)¨*°f¸dÀGÓL�
ñ %1°Ó$9Ð!Ü-2°6¸4Ó-@Ð! 'Ñ*à-7Ð)Ñ*à�r@   Fºr‹   NN)rC  rZ  r„   r  )TTrñ   rð   rG  )r,  rq   r-  r8  r_   rY  rg   Úcopy_ru   rp   Úcloner9   r
  r�   r?  r=  rç   r>  rS  )*r´   rN  r&  rO  rP  r_  rC  Úb_chunk_dimrt  Úz_cache_shaperk  Úz_cache_slicerÚz_cache_rotatedÚi_rangeÚi_1Úi_2Úinitial_offsetsÚ
after_halfrb  Úafter_half_offsetsÚcombined_range_with_offsetsrz   ÚoffsetÚ	z_chunk_bÚ
mask_chunkÚz_chunk_slicerÚz_cache_offsetrI  Úx_chunkÚ	z_chunk_gÚg_chunkÚz_slicerr  rv   rý   rp  rV  rc  rq  rr  rs  rh  s*   f&&f&                              @@@@@@@r=   Ú_inference_forwardÚ6EsmFoldTriangleMultiplicativeUpdate._inference_forwardU  s°  ÿø€ ðb Š<Ø—:‘:˜aŸg™g c r˜lÓ+ˆDà�~‰~˜bÓ!ˆ÷	÷ 	ñ 	ñB ˜q¨°dÔ;ˆàÓ)Ø—‘˜•ˆAØ˜!•V˜a !�e•^ˆFØˆGØˆGØ%)§^§^ ^™'¸ˆKò6õ÷ò ô8 ! §¡›MˆMØ%+ˆM˜'Ñ"Ø—k‘k -Ó0ˆGÙ)¨'Ó2ˆNÜ&+¨A¨vÓ&6ˆN˜7Ñ#Ø�M‰M˜!Õ+Ô,Ø#ˆOô
 œ5  FÐ,>Ó?Ó@ˆGÜ9<¸WÈbÅkÐU[ÐT\ÕF\Ô9]Ô^Ñ9]©X¨S˜sŸy˜yÑ9]ˆOÑ^Üœe F¨AÐ/AÓBÓCˆJÙ>HÓ!I¹j¸Ó"4¹jÐÐ!IÜ*-¨g¸
Õ.BÀOÐVhÕDhÓ*iÐ'Ü8‘	��6ß&¨1°¬;Ù+¨GÓ7�GØ&*�Oá(¨¨A¨q°6­z¸;ÓG�	Ù)¨$°°1°vµ:¸{ÓK�
à%ŸO™OÓ-�	Ø 'Ô)Ù ,¨Q°°1°vµ:¸wÓ G‘I÷
 +Ù)5°iÓ)@˜Ü27¸¸6Ó2B˜ wÑ/Ù4@ÀÈ!ÈQÐQWÍZÐY`Ó4a˜	 .Ò1à)*¨V­˜Ù$0°¸.È.Ð[aÕJaÐcjÓ$k˜	á,¨Y¸
ÀeÐUZÔ[�ØäŸ,š, q¨'Ó2�Ü,¨W°iÓ@�Ø×-Ñ-¨gÓ6�ØŸ-™-¨Ó0�ñ )¨¨A¨q°6­z¸7ÓC�	ØŸ-™-¨×(:Ñ(:¸9Ó(EÓF�Ø× Ñ Ô"Øà˜7Õ"�ñ (¨›?�Ü$)¨!¨Q°­ZÓ$8�˜Ñ!ßØ�h—K 7Õ*–Kà")�A�h”Kñ] 9ðx ˆñ # 1¨E°5Ó9ˆAÜ—’˜Q Ó"ˆAØ×#Ñ# AÓ&ˆAØ—‘˜aÓ ˆAØ—‘˜aÓ ˆAØ�J‰JŒLØ��FˆAßØ�Q•�ð ˆð �àˆùóA _ùâ!Is   ÅOÅ5Oc                ó–   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[RS[R,          RS[ P                  /# )r!   rN  r&  Nr  Ú_add_with_inplacerD  r^   )r9   r`   r¥   r�   )r;   r<   s   "€r=   r>   rE  8  s\   ø€ ÷ 4ñ 4á�<‰<ð4ñ �l‰l˜TÕ!ð4ñ ð	4ñ
  ð4ñ ! 4�Zð4ñ 
�‰ñ4r@   c                óÊ  € V'       d   V P                  VVVVR7      pV# Vf   VP                  VP                  RR 4      pVP                  R4      pV P	                  V4      pTpWpP                  V P                  V4      4      ,          pWpP                  V4      ,          pTpW€P                  V P                  V4      4      ,          pW€P                  V4      ,          pVP                  P                  R8w  d   VP                  P                  MRp	\        V	4      '       dN   \        V	RR7      ;_uu_ 4        V P                  VP                  4       VP                  4       4      pRRR4       MV P                  Wx4      p??V P!                  X4      pV P#                  V4      pV P                  V P%                  V4      4      p
Wj,          pV#   + '       g   i     L^; i)z›
Args:
    x:
        [*, N_res, N_res, C_z] input tensor
    mask:
        [*, N_res, N_res] input mask
Returns:
    [*, N_res, N_res, C_z] output tensor
)rO  rP  NÚmpsÚcpuFrÆ   r„   )r�  r,  rq   r-  r>  ré   r:  r9  r<  r;  rd   ÚtyperS   r   rK  ra   r?  r=  rç   )r´   rN  r&  r  r�  rD  rv   rC  r  rP   rý   s   &&&&&&     r=   rÎ   Ú+EsmFoldTriangleMultiplicativeUpdate.forward8  s‰  € ÷" Ø×'Ñ'ØØØ#6Ø*ð	 (ó ˆAð ˆHàŠ<Ø—:‘:˜aŸg™g c r˜lÓ+ˆDà�~‰~˜bÓ!ˆà×Ñ˜qÓ!ˆØˆØ—‘˜TŸ_™_¨QÓ/Ó0Õ0ˆØ—‘ Ó"Õ"ˆØˆØ—‘˜TŸ_™_¨QÓ/Ó0Õ0ˆØ—‘ Ó"Õ"ˆà'(§x¡x§}¡}¸Ô'=�a—h‘h—m’mÀ5ˆÜ˜;×'Ò'Ü¨KÀ×GÖGØ×-Ñ-¨a¯g©g«i¸¿¹»ÓC�÷ HÐGð ×)Ñ)¨!Ó/ˆAàˆqØ×Ñ Ó"ˆØ�M‰M˜!ÓˆØ�L‰L˜Ÿ™ qÓ)Ó*ˆØ�Eˆàˆ÷ H×Gús   Ä;/GÇG"	)
r8  r>  r?  r:  r9  r<  r;  rç   r=  ré   r  rL   )NNT)NFFé   )rB   rC   rD   rE   rF   r±   rK  r�  rÎ   rH   rI   r·   r¸   s   @@r=   r5  r5  %  s5   ù‡ € ñ÷$÷"0ò 0÷4aò a÷F4÷ 4ò 4r@   r5  c                   ó^   a a€ ] tR tRt oRt]P                  ! 4       V 3R l4       tRtVt	V ;t
# )ÚEsmFoldPreTrainedModelio  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
c                ó4  <€ \        V\        4      '       EdQ   \        P                  ! 4       ;_uu_ 4        VP                  e)   VP	                  VP
                  VP                  4       EMòVP                  R8X  dk   VP
                  P                  pR\        ^V^,          4      ,          p\        P                  ! V4      p\        P                  ! VP
                  VR7       EMwVP                  R8X  dj   VP
                  P                  pR\        ^V^,          4      ,          p\        P                  ! V4      p\        P                  ! VP
                  VR7       MýVP                  R8X  d$   \        P                  ! VP
                  ^R7       MÉVP                  R	8X  dT   \        P                  ! VP
                  4       VP                  '       d!   \        P                  ! VP                  4       MeVP                  R
8X  d$   \        P                   ! VP
                  RR7       M1VP                  R8X  d!   \        P                  ! VP
                  4       RRR4       R# \        V\"        4      '       d&   Rp\        P$                  ! VP&                  V4       R# \        V\(        4      '       Ed»   \        P                  ! VP*                  P,                  P
                  4       \        P                  ! VP*                  P,                  P                  4       \        P                  ! VP.                  P,                  P
                  4       \        P                  ! VP.                  P,                  P                  4       \        P                  ! VP0                  P2                  P4                  P
                  4       \        P                  ! VP0                  P2                  P4                  P                  4       \        P                  ! VP6                  P2                  P4                  P
                  4       \        P                  ! VP6                  P2                  P4                  P                  4       \        P                  ! VP8                  P:                  P
                  4       \        P                  ! VP8                  P:                  P                  4       \        P                  ! VP<                  P>                  P
                  4       \        P                  ! VP@                  P:                  P
                  4       \        P                  ! VP@                  P:                  P                  4       \        P                  ! VPB                  PD                  R,          P
                  4       \        P                  ! VPB                  PD                  R,          P                  4       \        P                  ! VPF                  PD                  R,          P
                  4       \        P                  ! VPF                  PD                  R,          P                  4       R# \H        SV `•  V4       R#   + '       g   i     R# ; i)zInitialize the weightsNrª   ç      ð?)Ústdr«   g       @r¬   )Úgainr­   r®   r  )Únonlinearityr¯   gËab€ˆRá?rð   )&r“   rž   r9   r²   r¤   rÁ   r¢   r£   rq   rr   rô   rõ   Únormal_Úxavier_uniform_Úzeros_Úones_Úkaiming_normal_ÚEsmFoldInvariantPointAttentionÚ	constant_Úhead_weightsÚ#EsmFoldTriangularSelfAttentionBlockÚ
tri_mul_inr=  Útri_mul_outÚtri_att_startr  ræ   Útri_att_endÚsequence_to_pairÚo_projÚpair_to_sequencer  Úseq_attentionÚmlp_seqÚmlpÚmlp_pairr°   Ú_init_weights)r´   Úmodulerq   Úscaler›  Úsoftplus_inverse_1rµ   s   &&    €r=   r²  Ú$EsmFoldPreTrainedModel._init_weightsv  sé  ø€ ô �fœm×,Ó,Ü—’—•Ø—>‘>Ò-Ø—N‘N 6§=¡=°&·+±+Ö>Ø—[‘[ IÔ-Ø"ŸM™M×/Ñ/�EØ¤# a¨¨q­Ó"2Õ2�EÜŸ)š) EÓ*�CÜ—L’L §¡°C×8Ø—[‘[ FÔ*Ø"ŸM™M×/Ñ/�EØ¤# a¨¨q­Ó"2Õ2�EÜŸ)š) EÓ*�CÜ—L’L §¡°CÖ8Ø—[‘[ HÔ,Ü×(Ò(¨¯©¸QÖ?Ø—[‘[ HÔ,Ü—K’K §¡Ô.Ø—{—{�{ÜŸ
š
 6§;¡;Ô/øØ—[‘[ HÔ,Ü×(Ò(¨¯©ÀXÖNØ—[‘[ GÔ+Ü—K’K §¡Ô.÷- !‘ô. ˜Ô >×?Ò?Ø!2ÐÜ�NŠN˜6×.Ñ.Ð0BÖCÜ˜Ô C×DÓDÜ�KŠK˜×)Ñ)×2Ñ2×9Ñ9Ô:Ü�KŠK˜×)Ñ)×2Ñ2×7Ñ7Ô8Ü�KŠK˜×*Ñ*×3Ñ3×:Ñ:Ô;Ü�KŠK˜×*Ñ*×3Ñ3×8Ñ8Ô9Ü�KŠK˜×,Ñ,×0Ñ0×9Ñ9×@Ñ@ÔAÜ�KŠK˜×,Ñ,×0Ñ0×9Ñ9×>Ñ>Ô?Ü�KŠK˜×*Ñ*×.Ñ.×7Ñ7×>Ñ>Ô?Ü�KŠK˜×*Ñ*×.Ñ.×7Ñ7×<Ñ<Ô=ä�KŠK˜×/Ñ/×6Ñ6×=Ñ=Ô>Ü�KŠK˜×/Ñ/×6Ñ6×;Ñ;Ô<Ü�KŠK˜×/Ñ/×6Ñ6×=Ñ=Ô>Ü�KŠK˜×,Ñ,×3Ñ3×:Ñ:Ô;Ü�KŠK˜×,Ñ,×3Ñ3×8Ñ8Ô9Ü�KŠK˜Ÿ™×*Ñ*¨2Õ.×5Ñ5Ô6Ü�KŠK˜Ÿ™×*Ñ*¨2Õ.×3Ñ3Ô4Ü�KŠK˜Ÿ™×+Ñ+¨BÕ/×6Ñ6Ô7Ü�KŠK˜Ÿ™×+Ñ+¨BÕ/×4Ñ4Ö5ä‰GÑ! &Ö)÷] !—�ús   ¶F"ZÇBZÚZ	rA   )rB   rC   rD   rE   rF   r9   r²   r²  rH   rI   r·   r¸   s   @@r=   r˜  r˜  o  s)   ù‡ € ñð ‡]‚]ƒ_ô1*ó ÷1*ð 1*r@   r˜  c                   ó@   a a€ ] tR tRt oRV 3R lltRR ltRtVtV ;t# )ÚEsmFoldSelfAttentioni«  c                óf  <€ \         SV `  4        WV,          8X  g   Q hWn        W n        W0n        \
        P                  ! W^,          RR7      V n        \
        P                  ! WRR7      V n        W@n	        V'       dp   \
        P                  ! W4      V n
        \        P                  ! V P                  P                  4       \        P                  ! V P                  P                  4       V P                  R,          V n        \        P                  ! V P                  P                  4       R# )é   Fr©   TNç      à¿)r°   r±   Ú	embed_dimÚ	num_headsÚ
head_widthr¾   ÚLinearÚprojr¬  ÚgatedÚg_projr£   r   rÁ   r¡  r¢   Úrescale_factor)r´   r¼  r½  r¾  rÁ  rµ   s   &&&&&€r=   r±   ÚEsmFoldSelfAttention.__init__¬  sÁ   ø€ Ü‰ÑÔØ¨
Õ2Ô2Ð2Ð2à"ŒØ"ŒØ$Œä—I’I˜i°Q­¸UÔCˆŒ	Ü—i’i 	¸4Ô@ˆŒØŒ
ßÜŸ)š) IÓ9ˆDŒKÜ�KŠK˜Ÿ™×*Ñ*Ô+Ü�JŠJ�t—{‘{×'Ñ'Ô(à"Ÿo™o¨tÕ3ˆÔä�Š�D—K‘K×$Ñ$Ö%r@   c                óB  € V P                  V4      P                  ! . VP                  R,          OV P                  NRN5!  pVP	                  ^ ^^^4      pVP                  ^RR7      w  rgpV P                  V,          p\        P                  ! RWg4      p	Ve   W“P	                  ^ ^^^4      ,           p	Ve.   VR,          pV	P                  VR8H  \        P                  ) 4      p	\        P                  P                  V	RR7      p	\        P                  ! RW˜4      p
V
P                  ! . V
P                  R,          ORN5!  p
V P                   '       d'   V P#                  V4      P%                  4       V
,          p
V P'                  V
4      p
W©P	                  ^ ^^^4      3# )a¤  
Basic self attention with optional mask and external pairwise bias. To handle sequences of different lengths,
use mask.

Inputs:
    x: batch of input sequences (.. x L x C) mask: batch of boolean masks where 1=valid, 0=padding position (..
    x L_k) bias: batch of scalar pairwise attention biases (.. x Lq x Lk x num_heads)

Outputs:
  sequence projection (B x L x embed_dim), attention maps (B x L x L x num_heads)
:Nr!   NrÓ   z...qc,...kc->...qkFz...hqk,...hkc->...qhcr„   )r)  NN)rÀ  rò   rq   r½  rŒ   ÚchunkrÃ  r9   ÚeinsumÚmasked_fillÚnpr  r¾   rÊ   rÔ   r…   rÁ  rÂ  ré   r¬  )r´   rv   r&  r¢   Úindicesr|   rö   ri   rš   rC  Úys   &&&&&      r=   rÎ   ÚEsmFoldSelfAttention.forwardÀ  s\  € ð �I‰I�a‹L×ÒÐ?˜qŸw™w r�{Ð?¨D¯N©NÐ?¸BÓ?ˆØ�I‰I�a˜˜A˜qÓ!ˆØ—'‘'˜! �'Ó$‰ˆˆaà×Ñ !Õ#ˆÜ�LŠLÐ-¨qÓ4ˆð ÒØ—L‘L  A q¨!Ó,Õ,ˆAð ÒØ˜Õ&ˆDØ—‘˜d e™m¬b¯f©f¨WÓ5ˆAä�M‰M×!Ñ! !¨Ð!Ó,ˆä�LŠLÐ0°!Ó7ˆØ�IŠIÐ'�q—w‘w˜r•{Ð' BÓ'ˆà�:�:ˆ:Ø—‘˜A“×&Ñ&Ó(¨1Õ,ˆAØ�K‰K˜‹Nˆà—)‘)˜A˜q ! QÓ'Ð'Ð'r@   )r¼  rÂ  rÁ  r¾  r½  r¬  rÀ  rÃ  )F)NNNrÐ   r¸   s   @@r=   r¸  r¸  «  s   ù‡ € ÷&÷(&(ô &(r@   r¸  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚEsmFoldDropoutié  zd
Implementation of dropout with the ability to share the dropout mask along a particular dimension.
c                óF   <€ V ^8„  d   QhRS[ RS[S[S[,          ,          /# )r!   ÚrÚ	batch_dim)ra   r�   r_   )r;   r<   s   "€r=   r>   ÚEsmFoldDropout.__annotate__î  s#   ø€ ÷ *ñ *™%ð *©C±$±sµ)­Oñ *r@   c                ó¸   <€ \         SV `  4        Wn        \        V\        4      '       d   V.pW n        \        P                  ! V P                  4      V n        R # rL   )	r°   r±   rÐ  r“   r�   rÑ  r¾   ÚDropoutÚdropout)r´   rÐ  rÑ  rµ   s   &&&€r=   r±   ÚEsmFoldDropout.__init__î  s@   ø€ Ü‰ÑÔàŒÜ�i¤×%Ò%Ø"˜ˆIØ"ŒÜ—z’z $§&¡&Ó)ˆŽr@   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r!   rv   r^   rû   )r;   r<   s   "€r=   r>   rÒ  ÷  s#   ø€ ÷ 3ñ 3™Ÿ™ð 3©%¯,©,ñ 3r@   c                óÂ   € \        VP                  4      pV P                  e   V P                   F  p^W#&   K	  	  WP                  VP	                  V4      4      ,          # rL   )r_   rq   rÑ  rÕ  r,  )r´   rv   rq   Úbds   &&  r=   rÎ   ÚEsmFoldDropout.forward÷  sI   € Ü�Q—W‘W“ˆØ�>‰>Ò%Ø—n”n�Ø�“	ñ %à—<‘< §
¡
¨5Ó 1Ó2Õ2Ð2r@   )rÑ  rÕ  rÐ  ©
rB   rC   rD   rE   rF   r±   rÎ   rH   rI   r·   r¸   s   @@r=   rÎ  rÎ  é  s#   ù‡ € ñ÷*ó *÷3÷ 3ð 3r@   rÎ  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚEsmFoldSequenceToPairiÿ  c                ó”  <€ \         SV `  4        \        P                  ! V4      V n        \        P
                  ! W^,          RR7      V n        \        P
                  ! ^V,          VRR7      V n        \        P                  ! V P                  P                  4       \        P                  ! V P                  P                  4       R# )r!   Tr©   N)r°   r±   r¾   r   Ú	layernormr¿  rÀ  r¬  r£   r   r¢   )r´   Úsequence_state_dimÚ	inner_dimr7  rµ   s   &&&&€r=   r±   ÚEsmFoldSequenceToPair.__init__   sy   ø€ Ü‰ÑÔäŸšÐ&8Ó9ˆŒÜ—I’IÐ0¸aµ-ÀdÔKˆŒ	Ü—i’i  I¥Ð/AÈÔMˆŒä�Š�D—I‘I—N‘NÔ#Ü�Š�D—K‘K×$Ñ$Ö%r@   c                ó\  € \        VP                  4      ^8X  g   Q hV P                  V4      pV P                  V4      pVP	                  ^RR7      w  r4VR,          VR,          ,          pVR,          VR,          ,
          p\
        P                  ! WV.RR7      pV P                  V4      pV# )zŸ
Inputs:
  sequence_state: B x L x sequence_state_dim

Output:
  pairwise_state: B x L x L x pairwise_state_dim

Intermediate state:
  B x L x L x 2*inner_dim
rÓ   r„   )r)  Nr)  r)  )r)  r)  Nr)  )rl   rq   rß  rÀ  rÆ  r9   Úcatr¬  )r´   Úsequence_staterÕ   rö   ri   ÚprodÚdiffrv   s   &&      r=   rÎ   ÚEsmFoldSequenceToPair.forward
  s˜   € ô �>×'Ñ'Ó(¨AÔ-Ð-Ð-à�N‰N˜>Ó*ˆØ�I‰I�a‹LˆØ�w‰w�q˜bˆwÓ!‰ˆà�Õ ! MÕ"2Õ2ˆØ�Õ ! MÕ"2Õ2ˆä�IŠI�t�l¨Ô+ˆØ�K‰K˜‹Nˆàˆr@   )rß  r¬  rÀ  rÐ   r¸   s   @@r=   rÝ  rÝ  ÿ  s   ù‡ € õ&÷ò r@   rÝ  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚEsmFoldPairToSequencei%  c                ó”   <€ \         SV `  4        \        P                  ! V4      V n        \        P
                  ! WR R7      V n        R# )Fr©   N)r°   r±   r¾   r   rß  r¿  r  )r´   r7  r½  rµ   s   &&&€r=   r±   ÚEsmFoldPairToSequence.__init__&  s2   ø€ Ü‰ÑÔäŸšÐ&8Ó9ˆŒÜ—i’iÐ 2ÀEÔJˆŽr@   c                ó‚   € \        VP                  4      ^8X  g   Q hV P                  V4      pV P                  V4      pV# )zj
Inputs:
  pairwise_state: B x L x L x pairwise_state_dim

Output:
  pairwise_bias: B x L x L x num_heads
)rl   rq   rß  r  )r´   Úpairwise_staterN  Úpairwise_biass   &&  r=   rÎ   ÚEsmFoldPairToSequence.forward,  s>   € ô �>×'Ñ'Ó(¨AÔ-Ð-Ð-Ø�N‰N˜>Ó*ˆØŸ™ A›ˆØÐr@   )rß  r  rÐ   r¸   s   @@r=   rê  rê  %  s   ù‡ € õK÷ò r@   rê  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚEsmFoldResidueMLPi:  c           	     ó(  <€ \         SV `  4        \        P                  ! \        P                  ! V4      \        P
                  ! W4      \        P                  ! 4       \        P
                  ! W!4      \        P                  ! V4      4      V n        R # rL   )	r°   r±   r¾   Ú
Sequentialr   r¿  ÚReLUrÔ  r°  )r´   r¼  rá  rÕ  rµ   s   &&&&€r=   r±   ÚEsmFoldResidueMLP.__init__;  sW   ø€ Ü‰ÑÔä—=’=Ü�LŠL˜Ó#Ü�IŠI�iÓ+Ü�GŠG‹IÜ�IŠI�iÓ+Ü�JŠJ�wÓó
ˆŽr@   c                ó0   € WP                  V4      ,           # rL   ©r°  )r´   rv   s   &&r=   rÎ   ÚEsmFoldResidueMLP.forwardF  s   € Ø—8‘8˜A“;�Ðr@   rø  r‘   rÐ   r¸   s   @@r=   rò  rò  :  s   ù‡ € ÷	
÷ò r@   rò  c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )r¦  iJ  c                óŠ  <€ \         SV `  4        Wn        VP                  pVP                  pW!P
                  ,          pW1P                  ,          p\        P                  ! V4      V n	        \        W#^,          V4      V n        \        W44      V n        \        W$VP
                  RR7      V n        \!        VRR7      V n        \!        VRR7      V n        \'        W1P                  VRRR7      V n        \'        W1P                  VRRR7      V n        \-        V^V,          VP.                  R7      V n        \-        V^V,          VP.                  R7      V n        \        P4                  ! VP.                  4      V n        \9        VP.                  ^,          ^4      V n        \9        VP.                  ^,          ^4      V n        R# )	r!   T)rÁ  )r8  Fr1  )r  r  )rÕ  N)r°   r±   r@  rà  r7  Úsequence_head_widthÚpairwise_head_widthr¾   r   Úlayernorm_1rÝ  r«  rê  r­  r¸  r®  r5  r¨  r§  r  r©  rª  rò  rÕ  r¯  r±  rÔ  ÚdroprÎ  Úrow_dropÚcol_drop)r´   r@  rà  r7  Úsequence_num_headsÚpairwise_num_headsrµ   s   &&    €r=   r±   Ú,EsmFoldTriangularSelfAttentionBlock.__init__K  s|  ø€ Ü‰ÑÔØŒà#×6Ñ6ÐØ#×6Ñ6ÐØ/×3MÑ3MÕMÐØ/×3MÑ3MÕMÐäŸ<š<Ð(:Ó;ˆÔä 5Ð6HÐ`aÕJaÐcuÓ vˆÔÜ 5Ð6HÓ ]ˆÔä1Ø°F×4NÑ4NÐVZô
ˆÔô ?¸vÐQUÔVˆÔÜ=¸fÐPUÔVˆŒä5Ø× :Ñ :Ð<NÐTWÐbfô
ˆÔô 4Ø× :Ñ :Ð<NÐTWÐbgô
ˆÔô )Ð);¸QÐASÕ=SÐ]c×]kÑ]kÔlˆŒÜ)Ð*<¸aÐBTÕ>TÐ^d×^lÑ^lÔmˆŒä—J’J˜vŸ~™~Ó.ˆŒ	Ü& v§~¡~¸Õ'9¸1Ó=ˆŒÜ& v§~¡~¸Õ'9¸1Ó=ˆŽr@   c           	     óž  € \        VP                  4      ^8w  d#   \        R\        VP                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      hVe=   \        VP                  4      ^8w  d#   \        R\        VP                  4       R24      hVP                  w  rgpVP                  ^,          p	W€P                  P                  8w  d'   \        RV RV P                  P                   R24      hW�P                  P
                  8w  d'   \        R	V	 RV P                  P
                   R24      hWbP                  ^ ,          8w  d$   \        R
V RVP                  ^ ,           R24      hWrP                  ^,          8w  g   WrP                  ^,          8w  d8   \        RV RVP                  ^,           RVP                  ^,           R24      hV P                  V4      p
V P                  V4      pV P                  W³V
R7      w  r¼WP                  V4      ,           pV P                  V4      pW P                  V4      ,           pVe(   VP                  ^4      VP                  ^4      ,          MRpW P                  V P                  W-R7      4      ,           pW P                  V P!                  W-R7      4      ,           pW P                  V P#                  W-VR7      4      ,           pW P                  V P%                  W-VR7      4      ,           pV P'                  V4      pW3# )zú
Inputs:
  sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim mask: B x L boolean
  tensor of valid positions

Output:
  sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim
z,`sequence_state` should be a 3d-tensor, got ú dims.z,`pairwise_state` should be a 4d-tensor, got Nú"`mask` should be a 2d-tensor, got zR`sequence_state` last dimension should be equal to `self.sequence_state_dim`. Got ú != Ú.zR`pairwise_state` last dimension should be equal to `self.pairwise_state_dim`. Got zD`sequence_state` and `pairwise_state` have inconsistent batch size: zI`sequence_state` and `pairwise_state` have inconsistent sequence length: z or )r&  r¢   ©r&  )r&  r  )rl   rq   r³   r@  rà  r7  r­  rþ  r®  rÿ  r¯  r«  r-  r   r¨  r  r§  r©  rª  r±  )r´   rå  rî  r&  r  Ú,_EsmFoldTriangularSelfAttentionBlock__kwargsrÑ  Úseq_dimrà  r7  r¢   rË  rb  Útri_masks   &&&&&,        r=   rÎ   Ú+EsmFoldTriangularSelfAttentionBlock.forwardm  s)  € ô ˆ~×#Ñ#Ó$¨Ô)ÜÐKÌCÐP^×PdÑPdÓLeÐKfÐflÐmÓnÐnÜˆ~×#Ñ#Ó$¨Ô)ÜÐKÌCÐP^×PdÑPdÓLeÐKfÐflÐmÓnÐnØÒ¤ D§J¡J£°1Ô 4ÜÐAÄ#ÀdÇjÁjÃ/ÐARÐRXÐYÓZÐZà1?×1EÑ1EÑ.ˆ	Ð.Ø+×1Ñ1°!Õ4Ðà§¡×!?Ñ!?Ô?ÜØdØ%Ð& d¨4¯;©;×+IÑ+IÐ*JÈ!ðMóð ð §¡×!?Ñ!?Ô?ÜØdØ%Ð& d¨4¯;©;×+IÑ+IÐ*JÈ!ðMóð ð ×,Ñ,¨QÕ/Ô/ÜØVÐW`ÐVaÐaeØ!×'Ñ'¨Õ*Ð+¨1ð.óð ð ×*Ñ*¨1Õ-Ô-°×<PÑ<PÐQRÕ<SÔ1SÜØ[Ð\cÐ[dÐdhØ!×'Ñ'¨Õ*Ð+¨4°×0DÑ0DÀQÕ0GÐ/HÈðKóð ð ×$Ñ$ ^Ó4ˆð ×Ñ˜^Ó,ˆØ×!Ñ! !°TÐ!Ó:‰ˆØ'¯)©)°A«,Õ6ˆØŸ™ nÓ5ˆð (×*?Ñ*?ÀÓ*OÕOˆð =AÒ<L�4—>‘> !Ó$ t§~¡~°aÓ'8Ö8ÐRVˆØ'¯-©-¸×8HÑ8HÈÐ8HÓ8gÓ*hÕhˆØ'¯-©-¸¿¹È¸Ó8fÓ*gÕgˆØ'¯-©-Ø×Ñ˜~ÈÐÓTó+
õ 
ˆð (¯-©-Ø×Ñ˜^ÀzÐÓRó+
õ 
ˆð
 Ÿ™ ~Ó6ˆàÐ-Ð-r@   )r  r@  rÿ  rþ  r±  r¯  r­  r   r®  r«  rª  r©  r§  r¨  )NNrÐ   r¸   s   @@r=   r¦  r¦  J  s   ù‡ € õ >÷DB.ô B.r@   r¦  c                   ó6   a € ] tR tRt o RR ltR tR tRtV tR# )ÚEsmCategoricalMixturei²  c                óâ   € Wn         \        P                  ! W4V^,           V P                   P                  V P                   P                  R7      pVRR VR,          ,           ^,          V n        R# )r‹   )rd   rc   Nrv  r„   )Úlogitsr9   Úlinspacerd   rc   Úv_bins)r´   ÚparamÚbinsrf  rg  s   &&&&&r=   r±   ÚEsmCategoricalMixture.__init__³  sQ   € àŒÜ�~Š~˜e¨$°­(¸4¿;¹;×;MÑ;MÐUY×U`ÑU`×UfÑUfÔgˆØ˜C˜R�y 4¨¥8Õ+¨qÕ0ˆŽr@   c                óT  € VP                  R4      V P                  R.VP                  ,          ,          ,
          P                  4       P	                  R4      pV P
                  P                  R4      p\        P                  ! W2P                  R4      RR7      P                  R4      # )r‹   NrÓ   r„   )
r-  r  ÚndimÚabsÚargminr  Úlog_softmaxr9   Útake_along_dimÚsqueeze)r´   ÚtrueÚ
true_indexÚnlls   &&  r=   Úlog_probÚEsmCategoricalMixture.log_prob¹  s~   € ð —n‘n RÓ(¨4¯;©;¸°vÀÇ	Á	Õ7IÕ+JÕJ×OÑOÓQ×XÑXÐY[Ó\ˆ
Ø�k‰k×%Ñ% bÓ)ˆÜ×#Ò# C×)=Ñ)=¸bÓ)AÀrÔJ×RÑRÐSUÓVÐVr@   c                ó–   € V P                   P                  R4      V P                  P                  ^4      ,          P	                  R4      # )r‹   r„   )r  rÔ   r  r-  r  )r´   s   &r=   ÚmeanÚEsmCategoricalMixture.meanÁ  s6   € Ø—‘×#Ñ# BÓ'¨$¯+©+×*?Ñ*?ÀÓ*BÕB×KÑKÈBÓOÐOr@   )r  r  N)é2   r   r‹   )	rB   rC   rD   rE   r±   r"  r%  rH   rI   rJ   s   @r=   r  r  ²  s   ø‡ € ô1òW÷Pð Pr@   r  c                 ó6   € \        WR 7      P                  4       # )©r  )r  r%  )r  r  s   &&r=   Úcategorical_lddtr*  Å  s   € ä  Ô3×8Ñ8Ó:Ð:r@   c                ó  € V f   R# \        V P                  4      ^8w  d#   \        R\        V P                  4       R24      hV P                  w  rV P                  ^4      P	                  WV4      pVP                  W,          V4      pV# )zµ
Helper to convert B x L mask of valid positions to axial mask used in row column attentions.

Input:
  mask: B x L tensor of booleans

Output:
  mask: B x L x L tensor of booleans
Nr  r  )rl   rq   r³   r-  Úexpandr…   )r&  rÑ  r  Úms   &   r=   Úget_axial_maskr.  Ê  sy   € ð ‚|Ùä
ˆ4�:‰:ƒ˜!ÔÜÐ=¼cÀ$Ç*Á*»oÐ=NÈfÐUÓVÐVØŸ™Ñ€IØ�‰�qÓ× Ñ  °WÓ=€AØ	�	‰	�)Õ% wÓ/€AØ€Hr@   c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )ÚEsmFoldRelativePositionià  c                óÔ   <€ \         SV `  4        VP                  V n        \        P
                  P                  ^V P                  ,          ^,           VP                  4      V n        R# ©r!   N)	r°   r±   Úposition_binsr  r9   r¾   Ú	Embeddingr7  Ú	embedding©r´   r@  rµ   s   &&€r=   r±   Ú EsmFoldRelativePosition.__init__á  sH   ø€ Ü‰ÑÔØ×(Ñ(ˆŒ	ô Ÿ™×+Ñ+¨A°·	±	­M¸AÕ,=¸v×?XÑ?XÓYˆŽr@   c                ó  € VP                   \        P                  8w  d   \        RVP                    R24      hVeB   VP                  VP                  8w  d'   \        RVP                   RVP                   R24      hVR,          VR,          ,
          pVP                  V P                  ) V P                  4      pW0P                  ,           ^,           pVe   VR,          VR,          ,          p^ W2R8H  &   V P                  V4      pV# )	z²
Input:
  residue_index: B x L tensor of indices (dtype=torch.long) mask: B x L tensor of booleans

Output:
  pairwise_state: B x L x L x pairwise_state_dim tensor of embeddings
z`residue_index` has dtype z, it should be `torch.long`.z5`residue_index` and `mask` have inconsistent shapes: r  r	  F)r)  Nr)  )r)  r)  N)rc   r9   Úlongr³   rq   Úclampr  r5  )r´   r1   r&  rç  r  s   &&&  r=   rÎ   ÚEsmFoldRelativePosition.forwardé  sô   € ð ×Ñ¤%§*¡*Ô,ÜÐ9¸-×:MÑ:MÐ9NÐNjÐkÓlÐlØÒ × 3Ñ 3°t·z±zÔ AÜØGÈ×H[ÑH[ÐG\Ð\`Ðae×akÑakÐ`lÐlmÐnóð ð ˜ZÕ(¨=¸Õ+DÕDˆØ�z‰z˜4Ÿ9™9˜* d§i¡iÓ0ˆØ—i‘iÕ !Õ#ˆàÒØ˜
Õ# d¨:Õ&6Õ6ˆDØ"#ˆD˜‘Ñà—‘ Ó%ˆØˆr@   )r  r5  rL   rÐ   r¸   s   @@r=   r0  r0  à  s   ù‡ € õZ÷ô r@   r0  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚEsmFoldAngleResnetBlocki  c                óô   <€ \         SV `  4        \        VP                  VP                  R R7      V n        \        VP                  VP                  RR7      V n        \        P                  ! 4       V n        R# ©r«   râ   r¯   N)	r°   r±   rž   Ú
resnet_dimÚlinear_1Úlinear_2r¾   rõ  r«   r6  s   &&€r=   r±   Ú EsmFoldAngleResnetBlock.__init__  sU   ø€ Ü‰ÑÔä% f×&7Ñ&7¸×9JÑ9JÐQWÔXˆŒÜ% f×&7Ñ&7¸×9JÑ9JÐQXÔYˆŒä—G’G“IˆŽ	r@   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r!   rC  r^   rû   )r;   r<   s   "€r=   r>   Ú$EsmFoldAngleResnetBlock.__annotate__  s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r@   c                óž   € TpV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           # rL   )r«   rA  rB  )r´   rC  Ú	s_initials   && r=   rÎ   ÚEsmFoldAngleResnetBlock.forward  sE   € Øˆ	à�I‰I�a‹LˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!Óˆà�}Ðr@   )rA  rB  r«   rÐ   r¸   s   @@r=   r=  r=    s   ù‡ € õ÷÷ ð r@   r=  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚEsmFoldAngleResneti  z&
Implements Algorithm 20, lines 11-14
c                ó  <€ \         SV `  4        Wn        \        VP                  VP
                  4      V n        \        VP                  VP
                  4      V n        \        P                  ! 4       V n
        \        VP                  4       F)  p\        V4      pV P                  P                  V4       K+  	  \        VP
                  VP                  ^,          4      V n        \        P"                  ! 4       V n        R# r2  )r°   r±   r@  rž   Úsequence_dimr@  Ú	linear_inÚlinear_initialr¾   Ú
ModuleListÚlayersru   Únum_resnet_blocksr=  ÚappendÚ
num_anglesÚ
linear_outrõ  r«   )r´   r@  rb  Úlayerrµ   s   &&  €r=   r±   ÚEsmFoldAngleResnet.__init__  s´   ø€ Ü‰ÑÔØŒä& v×':Ñ':¸F×<MÑ<MÓNˆŒÜ+¨F×,?Ñ,?À×ARÑARÓSˆÔä—m’m“oˆŒÜ�v×/Ñ/Ö0ˆAÜ+¨FÓ3ˆEØ�K‰K×Ñ˜uÖ%ñ 1ô (¨×(9Ñ(9¸6×;LÑ;LÈqÕ;PÓQˆŒä—G’G“IˆŽ	r@   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r!   rÕ   rG  r^   rî   )r;   r<   s   "€r=   r>   ÚEsmFoldAngleResnet.__annotate__-  s?   ø€ ÷ )!ñ )!™Ÿ™ð )!±%·,±,ð )!Á5ÉÏÉÑW\×WcÑWcÐIcÕCdñ )!r@   c           
     ó4  € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV P                   F  pV! V4      pK  	  V P                  V4      pV P	                  V4      pVP                  VP                  RR R,           4      pTp\        P                  ! \        P                  ! \        P                  ! V^,          RRR7      V P                  P                  R7      4      pW,          pWA3# )zÂ
Args:
    s:
        [*, C_hidden] single embedding
    s_initial:
        [*, C_hidden] single embedding as of the start of the StructureModule
Returns:
    [*, no_angles, 2] predicted angles
NT)rm   Úkeepdim)Úminr„   )r„   r!   )r«   rN  rM  rP  rT  rò   rq   r9   rõ   r:  r	  r@  Úepsilon)r´   rÕ   rG  ÚlÚunnormalized_sÚ
norm_denoms   &&&   r=   rÎ   ÚEsmFoldAngleResnet.forward-  së   € ð —I‘I˜iÓ(ˆ	Ø×'Ñ'¨	Ó2ˆ	Ø�I‰I�a‹LˆØ�N‰N˜1ÓˆØ�Mˆà—”ˆAÙ�!“ŠAñ ð �I‰I�a‹Lˆð �O‰O˜AÓˆð �F‰F�1—7‘7˜3˜B�< 'Õ)Ó*ˆàˆÜ—Z’ZÜ�KŠKÜ—	’	˜!˜Q�$ B°Ô5Ø—K‘K×'Ñ'ôó
ˆ
ð �NˆàÐ Ð r@   )r@  rP  rM  rN  rT  r«   rÛ  r¸   s   @@r=   rJ  rJ    s   ù‡ € ñõ÷ )!÷ )!ð )!r@   rJ  c                   óL   a a€ ] tR tRt oRtV 3R ltRV3R lR lltRtVtV ;t	# )r£  iY  z
Implements Algorithm 22.
c                ó  <€ \         SV `  4        Wn        VP                  pVP                  pVP
                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP
                  VP                  ,          p\        W$4      V n        \        V^V,          4      V n        VP                  VP                  ,          ^,          p\        W%4      V n        VP                  VP                  VP                  ,           ,          ^,          p\        W&4      V n        \        W1P                  4      V n        \"        P$                  ! \&        P(                  ! VP                  4      4      V n        VP                  W1P
                  ,           VP                  ^,          ,           ,          p\        WrRR7      V n        \"        P.                  ! RR7      V n        \"        P2                  ! 4       V n        R# )r!   r¯   râ   rÓ   Nr„   )r°   r±   r@  rL  Úpairwise_dimÚipa_dimÚ
hidden_dimÚnum_heads_ipar½  Únum_qk_pointsÚnum_v_pointsrž   rã   Ú	linear_kvÚlinear_q_pointsÚlinear_kv_pointsÚlinear_br¾   r¿   r9   rÂ   r¥  rT  ÚSoftmaxrÔ   ÚSoftplusÚsoftplus)	r´   r@  Úc_sÚc_zÚhcÚhpqÚhpkvÚconcat_out_dimrµ   s	   &&      €r=   r±   Ú'EsmFoldInvariantPointAttention.__init__^  sr  ø€ Ü‰ÑÔØŒà×!Ñ!ˆØ×!Ñ!ˆØ Ÿ.™.ˆŒØ×-Ñ-ˆŒØ#×1Ñ1ˆÔØ"×/Ñ/ˆÔð �^‰^˜f×2Ñ2Õ2ˆÜ% cÓ.ˆŒÜ& s¨A°­FÓ3ˆŒà×"Ñ" V×%9Ñ%9Õ9¸AÕ=ˆÜ,¨SÓ6ˆÔà×#Ñ# v×';Ñ';¸f×>QÑ>QÕ'QÕRÐUVÕVˆÜ -¨cÓ 8ˆÔä% c×+?Ñ+?Ó@ˆŒäŸLšL¬¯ª°V×5IÑ5IÓ)JÓKˆÔà×-Ñ-°·~±~Õ1EÈ×H[ÑH[Ð^_ÕH_Õ1_Õ`ˆÜ'¨À'ÔJˆŒä—z’z bÔ)ˆŒÜŸš›ˆŽr@   c                óÔ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[ P                  RS[RS[S[ P                  ,          R,          RS[ P                  /# )	r!   rÕ   rN  NrÐ  r&  Ú_offload_inferenceÚ_z_reference_listr^   )r9   r`   r   r¥   r   )r;   r<   s   "€r=   r>   Ú+EsmFoldInvariantPointAttention.__annotate__�  s}   ø€ ÷ [ñ [á�<‰<ð[ñ �<‰<˜$Õð[ñ ð	[ñ
 �l‰lð[ñ !ð[ñ $¡E§L¡LÕ1°DÕ8ð[ñ 
�‰ñ[r@   c           
     ó
  € V.pV P                  V4      pV P                  V4      pVP                  VP                  RR
 V P                  R
3,           4      pVP                  VP                  RR
 V P                  R
3,           4      p\
        P                  ! W€P                  R
R7      w  ršV P                  V4      p\
        P                  ! W»P                  R
,          ^,          R
R7      p\
        P                  ! VR
R7      pVR,          P                  V4      pVP                  VP                  RR V P                  V P                  ^3,           4      pV P                  V4      p\
        P                  ! WÌP                  R
,          ^,          R
R7      p\
        P                  ! VR
R7      pVR,          P                  V4      pVP                  VP                  RR V P                  R
^3,           4      p\
        P                  ! WÀP                  V P                  .RR7      w  rÞV P                  V^ ,          4      pV'       d?   \        P                   ! V^ ,          4      ^8X  g   Q hV^ ,          P#                  4       V^ &   VP$                  P&                  R8w  d   VP$                  P&                  MRp\)        V4      '       dg   \+        VRR7      ;_uu_ 4        \
        P,                  ! \/        VP1                  4       R4      \/        V	P1                  4       R4      4      pRRR4       M+\
        P,                  ! \/        VR4      \/        V	R4      4      pX\2        P4                  ! R^V P                  ,          ,          4      ,          pV\2        P4                  ! R4      \/        VR4      ,          ,          pVP7                  R4      VP7                  R4      ,
          pV^,          p\9        \
        P:                  ! VR
R7      4      pV P=                  V P>                  4      P                  ! R\A        VP                  RR 4      ,          R,           !  pV\2        P4                  ! R^V P                  R,          ^,          ,          ,          4      ,          pVV,          p\
        P8                  ! VR
R7      R,          pVP7                  R
4      VP7                  R4      ,          pV PB                  PD                  V^,
          ,          p\/        VR4      pVV,           pVVP7                  R4      ,           pV PG                  V4      p\
        P,                  ! VV
PI                  RR4      PK                  VPL                  R	7      4      PI                  RR4      p\O        V^4      p\
        P8                  ! VR,          \/        VR4      R,          ,          RR7      p\/        VR4      pVR,          PQ                  V4      p\O        \
        P4                  ! \
        P8                  ! V^,          R
R7      V PB                  PR                  ,           4      ^4      pVPT                  ! . VP                  RR OR
N^N5!  pV'       d&   V^ ,          PK                  VP$                  4      V^ &   \
        P,                  ! VPI                  RR4      V^ ,          PK                  VPL                  R	7      4      p\O        V^4      pV PW                  \
        PX                  ! V.\
        P:                  ! VR
R7      OVNVN5R
R7      PK                  V^ ,          PL                  R	7      4      pV#   + '       g   i     EL#; i)a  
Args:
    s:
        [*, N_res, C_s] single representation
    z:
        [*, N_res, N_res, C_z] pair representation
    r:
        [*, N_res] transformation object
    mask:
        [*, N_res] mask
Returns:
    [*, N_res, C_s] single representation update
NrÓ   r’  r“  FrÆ   rš  g      "@rÈ   r„   ).Nrð   )r‹   r   r!   rG  gUUUUUUÕ?r*  r+  éûÿÿÿ©r‹   )r„   r‹   r»  rñ   ).Nr)  r)  N)r‹   rº  r   r!   ©.Nr)  r)  )r!   r   rº  r‹   ).NN)-rã   ri  rò   rq   r½  r9   Úsplitre  rj  ÚstackÚapplyrg  rk  rh  rl  ÚsysÚgetrefcountr“  rd   r”  rS   r   r
  r�   ra   rô   rõ   r-  r	  Úunbindro  r¥  rl   r@  r  rÔ   ró   rÌ   rc   r†   Úinvert_applyr\  r…   rT  rä  )r´   rÕ   rN  rÐ  r&  rx  ry  rö   Úkvri   rš   Úq_ptsÚkv_ptsÚk_ptsÚv_ptsr  rP   rC  Úpt_attr¥  Úsquare_maskrú   Úo_ptÚ	o_pt_normÚo_pairs   &&&&&&&                  r=   rÎ   Ú&EsmFoldInvariantPointAttention.forward�  sÏ  € ð, ˆCˆð �M‰M˜!ÓˆØ�^‰^˜AÓˆð �F‰F�1—7‘7˜3˜B�< 4§>¡>°2Ð"6Õ6Ó7ˆð �W‰W�R—X‘X˜c˜r�] d§n¡n°bÐ%9Õ9Ó:ˆô �{Š{˜2Ÿ™°BÔ7‰ˆð ×$Ñ$ QÓ'ˆô —’˜E§;¡;¨r¥?°aÕ#7¸RÔ@ˆÜ—’˜E rÔ*ˆØ�)•×"Ñ" 5Ó)ˆð —
‘
˜5Ÿ;™; s¨Ð+¨t¯~©~¸t×?QÑ?QÐSTÐ.UÕUÓVˆð ×&Ñ& qÓ)ˆô —’˜V§\¡\°"Õ%5¸Õ%:ÀÔCˆÜ—’˜V¨Ô,ˆØ�9•×#Ñ# FÓ+ˆð —‘˜VŸ\™\¨#¨2Ð.°$·.±.À"ÀaÐ1HÕHÓIˆô —{’{ 6×,>Ñ,>À×@QÑ@QÐ+RÐXZÔ[‰ˆð �M‰M˜!˜A�$ÓˆçÜ—?’? 1 Q¥4Ó(¨AÔ-Ð-Ð-Ø�Q•4—8‘8“:ˆAˆa‰Dð ()§x¡x§}¡}¸Ô'=�a—h‘h—m’mÀ5ˆÜ˜;×'Ò'Ü¨KÀ×GÖGÜ—L’LÜ& q§w¡w£y°)Ó<Ü& q§w¡w£y°)Ó<ó�÷ HÐGô —’Ü" 1 iÓ0Ü" 1 iÓ0óˆAð
 	
ŒT�YŠY�s˜a $§/¡/Õ1Õ2Ó3Õ3ˆØ	ŒT�YŠY�wÓÔ"4°Q¸	Ó"BÕBÕBˆð —‘ Ó$ u§¡°rÓ':Õ:ˆØ˜•ˆô ”U—\’\ &¨bÔ1Ó2ˆØ—}‘} T×%6Ñ%6Ó7×<Ò<¸tÄcÈ&Ï,É,ÐWZÐXZÐJ[ÓF\Õ?\Ð_fÕ?fÑhˆØ#¤d§i¢i°°q¸D×<NÑ<NÐQTÕ<TÐWXÕ<XÕ7YÕ0ZÓ&[Õ[ˆØ˜,Õ&ˆô —’˜6 rÔ*¨dÕ3ˆà—n‘n RÓ(¨4¯>©>¸"Ó+=Õ=ˆØ—k‘k—o‘o¨°q­Õ9ˆô $ F¨IÓ6ˆà��JˆØ�×%Ñ% bÓ)Õ)ˆØ�L‰L˜‹Oˆô �LŠL˜˜AŸK™K¨¨BÓ/×2Ñ2¸¿¹Ð2ÓAÓB×LÑLÈRÐQSÓTˆô ˜q !Ó$ˆô �yŠyØÐ$Õ%Ô(:¸5À,Ó(OÐP_Õ(`Õ`Øô
ˆô " $¨Ó5ˆØ�Õ!×.Ñ.¨tÓ4ˆô '¤u§z¢z´%·)²)¸DÀ!½GÈÔ2LÈtÏ{É{×ObÑObÕ2bÓ'cÐefÓgˆ	ð �|Š|Ð4˜TŸZ™Z¨¨˜_Ð4¨bÐ4°!Ó4ˆçØ�Q•4—7‘7˜4Ÿ;™;Ó'ˆAˆa‰Dô —’˜aŸk™k¨"¨bÓ1°1°Qµ4·7±7ÀÇÁ°7Ó3IÓJˆô $ F¨AÓ.ˆð �O‰OÜ�IŠI�qÐIœ5Ÿ<š<¨°"Ô5ÐI°yÐIÀ&ÑIÈrÔR×UÑUÐ\]Ð^_Õ\`×\fÑ\fÐUÓgó
ˆð ˆ÷Y H×GÐGús   Ë#A]1Ý1^	)r@  r¥  re  rl  ri  rk  rT  rã   rj  r½  rg  rh  rÔ   ro  rU   rÛ  r¸   s   @@r=   r£  r£  Y  s"   ù‡ € ñõ!&÷F[÷ [ò [r@   r£  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚEsmFoldBackboneUpdatei  z"
Implements part of Algorithm 23.
c                ó^   <€ \         SV `  4        \        VP                  ^RR7      V n        R# )é   r¯   râ   N)r°   r±   rž   rL  r  r6  s   &&€r=   r±   ÚEsmFoldBackboneUpdate.__init__$  s$   ø€ Ü‰ÑÔä# F×$7Ñ$7¸ÀÔIˆŽr@   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  3,          /# )r!   rÕ   r^   rî   )r;   r<   s   "€r=   r>   Ú"EsmFoldBackboneUpdate.__annotate__)  s1   ø€ ÷ 
ñ 
™Ÿ™ð 
©%±·±¹e¿l¹lÐ0JÕ*Kñ 
r@   c                ó(   € V P                  V4      pV# )zZ
Args:
    [*, N_res, C_s] single representation
Returns:
    [*, N_res, 6] update vector
©r  )r´   rÕ   Úupdates   && r=   rÎ   ÚEsmFoldBackboneUpdate.forward)  s   € ð —‘˜Q“ˆàˆr@   r™  rÛ  r¸   s   @@r=   r’  r’    s   ù‡ € ñõJ÷

÷ 
ð 
r@   r’  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú%EsmFoldStructureModuleTransitionLayeri6  c                óB  <€ \         SV `  4        \        VP                  VP                  R R7      V n        \        VP                  VP                  R R7      V n        \        VP                  VP                  RR7      V n        \        P                  ! 4       V n	        R# r?  )
r°   r±   rž   rL  rA  rB  Úlinear_3r¾   rõ  r«   r6  s   &&€r=   r±   Ú.EsmFoldStructureModuleTransitionLayer.__init__7  sr   ø€ Ü‰ÑÔä% f×&9Ñ&9¸6×;NÑ;NÐU[Ô\ˆŒÜ% f×&9Ñ&9¸6×;NÑ;NÐU[Ô\ˆŒÜ% f×&9Ñ&9¸6×;NÑ;NÐU\Ô]ˆŒä—G’G“IˆŽ	r@   c                óÄ   € TpV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV# rL   )rA  r«   rB  rŸ  )r´   rÕ   rG  s   && r=   rÎ   Ú-EsmFoldStructureModuleTransitionLayer.forward@  sW   € Øˆ	Ø�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!Óˆà�Mˆàˆr@   )rA  rB  rŸ  r«   rÐ   r¸   s   @@r=   r�  r�  6  s   ù‡ € õ÷
ò 
r@   r�  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )Ú EsmFoldStructureModuleTransitioniM  c                óf  <€ \         SV `  4        Wn        \        P                  ! 4       V n        \        VP                  4       F)  p\        V4      pV P
                  P                  V4       K+  	  \        P                  ! VP                  4      V n        \        VP                  4      V n        R # rL   )r°   r±   r@  r¾   rO  rP  ru   Únum_transition_layersr�  rR  rÔ  Údropout_raterÕ  r   rL  rË   )r´   r@  rb  r]  rµ   s   &&  €r=   r±   Ú)EsmFoldStructureModuleTransition.__init__N  s{   ø€ Ü‰ÑÔØŒä—m’m“oˆŒÜ�v×3Ñ3Ö4ˆAÜ5°fÓ=ˆAØ�K‰K×Ñ˜qÖ!ñ 5ô —z’z &×"5Ñ"5Ó6ˆŒÜ# F×$7Ñ$7Ó8ˆŽr@   c                ó€   € V P                    F  pV! V4      pK  	  V P                  V4      pV P                  V4      pV# rL   )rP  rÕ  rË   )r´   rÕ   r]  s   && r=   rÎ   Ú(EsmFoldStructureModuleTransition.forwardZ  s;   € Ø—”ˆAÙ�!“ŠAñ ð �L‰L˜‹OˆØ�O‰O˜AÓˆàˆr@   )r@  rÕ  rË   rP  rÐ   r¸   s   @@r=   r¤  r¤  M  s   ù‡ € õ
9÷ò r@   r¤  c                   óN   a a€ ] tR tRt oV 3R ltRR ltR tR tR tRt	Vt
V ;t# )	ÚEsmFoldStructureModuleid  c                óà  <€ \         SV `  4        Wn        \        VP                  4      V n        \        VP                  4      V n        \        VP                  VP                  4      V n	        \        V4      V n        \        P                  ! VP                  4      V n        \        VP                  4      V n        \#        V4      V n        \'        V4      V n        \+        V4      V n        R # rL   )r°   r±   r@  r   rL  Úlayer_norm_src  Úlayer_norm_zrž   rM  r£  Úipar¾   rÔ  r§  Úipa_dropoutÚlayer_norm_ipar¤  Ú
transitionr’  Ú	bb_updaterJ  Úangle_resnetr6  s   &&€r=   r±   ÚEsmFoldStructureModule.__init__e  s°   ø€ Ü‰ÑÔØŒô & f×&9Ñ&9Ó:ˆÔÜ% f×&9Ñ&9Ó:ˆÔä& v×':Ñ':¸F×<OÑ<OÓPˆŒä1°&Ó9ˆŒäŸ:š: f×&9Ñ&9Ó:ˆÔÜ'¨×(;Ñ(;Ó<ˆÔä:¸6ÓBˆŒÜ.¨vÓ6ˆŒÜ.¨vÓ6ˆÖr@   c                óà  € VR,          pVf   VP                  VP                  RR 4      pV P                  V4      pV P                  VR,          4      pRpV'       dD   \        P
                  ! VR,          4      ^8X  g   Q hVR,          P                  4       VR&   V.pRpTpV P                  V4      p\        P                  ! VP                  RR VP                  VP                  V P                  RR7      p	. p
\        V P                  P                  4       EF€  pWPP!                  VVV	VVVR7      ,           pV P#                  V4      pV P%                  V4      pV P'                  V4      pV	P)                  V P+                  V4      4      p	\        \-        V	P/                  4       P1                  4       RR7      V	P3                  4       4      pVP5                  V P                  P6                  4      pV P9                  WX4      w  rÞV P;                  WÎV4      pV P=                  Wò4      pV	P5                  V P                  P6                  4      pRVP?                  4       R	VPA                  4       R
VRVRVRV/pV
PC                  V4       V	PE                  4       p	EKƒ  	  ??V'       d&   VR,          PG                  VP                  4      VR&   \I        \J        PL                  V
4      p
WZR&   V
# )ag  
Args:
    evoformer_output_dict:
        Dictionary containing:
            "single":
                [*, N_res, C_s] single representation
            "pair":
                [*, N_res, N_res, C_z] pair representation
    aatype:
        [*, N_res] amino acid indices
    mask:
        Optional [*, N_res] sequence mask
Returns:
    A dictionary of outputs
ÚsingleNrT  Úquat)Úfmt)rx  ry  )Úrot_matsÚquatsr"   r#   r$   r%   r&   r'   r„   )'r,  rq   r®  r¯  r‚  rƒ  r“  rM  r   Úidentityrc   rd   Útrainingru   r@  Ú
num_blocksr°  r±  r²  r³  Úcompose_q_update_vecr´  r   Úget_rotsÚget_rot_matsÚ	get_transÚscale_translationÚtrans_scale_factorrµ  r   r   Úto_tensor_7Úto_tensor_4x4rR  Ústop_rot_gradientrÌ   r•   r9   r€  )r´   Úevoformer_output_dictr,   r&  rx  rÕ   rN  Úz_reference_listrG  ÚrigidsÚoutputsrz   Úbackb_to_globalr$   r%   Úall_frames_to_globalÚpred_xyzÚscaled_rigidsÚpredss   &&&&&              r=   rÎ   ÚEsmFoldStructureModule.forward}  sÃ  € ð, " (Õ+ˆàŠ<à—:‘:˜aŸg™g c r˜lÓ+ˆDð ×Ñ˜aÓ ˆð ×ÑÐ3°FÕ;Ó<ˆàÐßÜ—?’?Ð#8¸Õ#@ÓAÀQÔFÐFÐFØ,AÀ&Õ,I×,MÑ,MÓ,OÐ! &Ñ)Ø !˜sÐØˆAð ˆ	Ø�N‰N˜1Óˆô —’Ø�G‰G�C�RˆLØ�G‰GØ�H‰HØ�M‰MØô
ˆð ˆÜ�t—{‘{×-Ñ-×.ˆAà—H‘HØØØØØ#5Ø"2ð ó õ ˆAð × Ñ  Ó#ˆAØ×#Ñ# AÓ&ˆAØ—‘ Ó"ˆAð ×0Ñ0°·±ÀÓ1BÓCˆFô
 $Ü &§/¡/Ó"3×"@Ñ"@Ó"BÈ$ÔOØ× Ñ Ó"óˆOð
 .×?Ñ?ÀÇÁ×@^Ñ@^Ó_ˆOð +/×*;Ñ*;¸AÓ*IÑ'Ðà#'×#@Ñ#@ÀÐZ`Ó#aÐ à×IÑIÐJ^ÓgˆHà"×4Ñ4°T·[±[×5SÑ5SÓTˆMð ˜-×3Ñ3Ó5Ø"Ð$8×$FÑ$FÓ$HØ%Ð':Ø˜&Ø˜XØ˜!ðˆEð �N‰N˜5Ô!à×-Ñ-Ó/‹Fñ_ /ðb ÐçØ,AÀ&Õ,I×,LÑ,LÈQÏXÉXÓ,VÐ! &Ñ)ä¤§¡¨WÓ5ˆØ�Ñàˆr@   c           	     ó°  € \        V R 4      '       gD   \        P                  ! \        P                  ! \
        P                  VVRR7      RR7      V n        \        V R4      '       gC   \        P                  ! \        P                  ! \
        P                  VRR7      RR7      V n	        \        V R4      '       gD   \        P                  ! \        P                  ! \
        P                  VVRR7      RR7      V n        \        V R4      '       gF   \        P                  ! \        P                  ! \
        P                  VVRR7      RR7      V n        R# R# )	Údefault_framesF)rc   rd   Úrequires_grad)Ú
persistentÚ	group_idx)rd   rÕ  Ú	atom_maskÚlit_positionsN)Úhasattrr¾   ÚBufferr9   rˆ   r   Ú!restype_rigid_group_default_framerÔ  Úrestype_atom14_to_rigid_groupr×  Úrestype_atom14_maskrØ  Ú$restype_atom14_rigid_group_positionsrÙ  )r´   Úfloat_dtyperd   s   &&&r=   Ú_init_residue_constantsÚ.EsmFoldStructureModule._init_residue_constantsî  s  € Ü�tÐ-×.Ò.Ü"$§)¢)Ü—’Ü%×GÑGØ%Ø!Ø"'ô	ð !ô#ˆDÔô �t˜[×)Ò)ÜŸYšYÜ—’Ü%×CÑCØ!Ø"'ôð
 !ôˆDŒNô �t˜[×)Ò)ÜŸYšYÜ—’Ü%×9Ñ9Ø%Ø!Ø"'ô	ð !ôˆDŒNô �t˜_×-Ò-Ü!#§¢Ü—’Ü%×JÑJØ%Ø!Ø"'ô	ð !ô"ˆDÖñ .r@   c                óz   € V P                  VP                  VP                  4       \        WW0P                  4      # rL   )rá  rc   rd   r   rÔ  )r´   rÐ  ÚalphaÚfs   &&&&r=   r   Ú/EsmFoldStructureModule.torsion_angles_to_frames  s-   € à×$Ñ$ U§[¡[°%·,±,Ô?ä'¨°!×5HÑ5HÓIÐIr@   c                óö   € V P                  VP                  4       P                  VP                  4       P                  4       \	        VVV P
                  V P                  V P                  V P                  4      # rL   )	rá  rÁ  rc   rd   r   rÔ  r×  rØ  rÙ  )r´   rÐ  rå  s   &&&r=   r   ÚDEsmFoldStructureModule.frames_and_literature_positions_to_atom14_pos  s]   € à×$Ñ$ Q§Z¡Z£\×%7Ñ%7¸¿¹»×9LÑ9LÔMÜ<ØØØ×ÑØ�N‰NØ�N‰NØ×Ñó
ð 	
r@   )rµ  rØ  r´  r@  rÔ  r×  r°  r±  r²  r®  r¯  rM  rÙ  r³  )NF)rB   rC   rD   rE   r±   rÎ   rá  r   r   rH   rI   r·   r¸   s   @@r=   r¬  r¬  d  s'   ù‡ € õ7ô0oòb'òRJ÷

ò 

r@   r¬  c                   óN   a a€ ] tR tRt oV 3R ltR tR t]R 4       tRt	Vt
V ;t# )ÚEsmFoldingTrunki*  c                ób  <€ \         SV `  4        Wn        VP                  pVP                  p\        V4      V n        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        ^V n        \        P                  ! V4      V n        \        P                  ! V4      V n        \        P"                  ! V P                  V4      V n        V P$                  P&                  ^ ,          P)                  4       P+                  4        \-        VP.                  4      V n        \        P0                  ! W!P.                  P2                  4      V n        \        P0                  ! W1P.                  P6                  4      V n        VP:                  V n        R# u upi )é   N)r°   r±   r@  rà  r7  r0  Úpairwise_positional_embeddingr¾   rO  ru   r¿  r¦  ÚblocksÚrecycle_binsr   Úrecycle_s_normÚrecycle_z_normr4  Úrecycle_distorÁ   ÚdetachÚzero_r¬  Ústructure_moduler¿  rL  Ú
trunk2sm_src  Ú
trunk2sm_zr  )r´   r@  rp  rq  rb  rµ   s   &&   €r=   r±   ÚEsmFoldingTrunk.__init__+  s2  ø€ Ü‰ÑÔØŒà×'Ñ'ˆØ×'Ñ'ˆä-DÀVÓ-LˆÔ*ä—m’mÔZ_Ð`f×`qÑ`qÔZrÓ$sÑZrÐUVÔ%HÈÖ%PÑZrÑ$sÓtˆŒàˆÔÜ Ÿlšl¨3Ó/ˆÔÜ Ÿlšl¨3Ó/ˆÔÜŸ\š\¨$×*;Ñ*;¸SÓAˆÔØ×Ñ×!Ñ! !Õ$×+Ñ+Ó-×3Ñ3Ô5ä 6°v×7NÑ7NÓ OˆÔÜŸ)š) C×)@Ñ)@×)MÑ)MÓNˆŒÜŸ)š) C×)@Ñ)@×)MÑ)MÓNˆŒà ×+Ñ+ˆŽùò %ts   Á%F,c                ó   € Wn         R # rL   )r  )r´   r  s   &&r=   Úset_chunk_sizeÚEsmFoldingTrunk.set_chunk_sizeB  s	   € ð
 %Žr@   c                óš  a € VP                   pTpTp	Vf   S P                  P                  pMV^ 8  d   \        R4      hV^,          pV 3R lp
TpT	p\        P
                  ! V4      p\        P
                  ! V4      p\        P                  ! VP                  RR RVR\        P                  / p\        V4       EFk  p\        VV^,
          8X  d   . M\        P                  ! 4       .4      ;_uu_ 4        S P                  VP                  4       4      P                  V4      pS P                  VP                  4       4      P                  V4      pVS P!                  VP                  4       4      P                  V4      ,          pV
! W�,           Wž,           WE4      w  r¼S P#                  RS P%                  V4      RS P'                  V4      /VVP)                  4       4      pTpTp\*        P-                  VR,          R,          R,          R	R
S P.                  4      pRRR4       EKn  	  VXR&   VVR&   V#   + '       g   i     EKŽ  ; i)aN  
Inputs:
  seq_feats: B x L x C tensor of sequence features pair_feats: B x L x L x C tensor of pair features residx: B
  x L long tensor giving the position in the sequence mask: B x L boolean tensor indicating valid residues

Output:
  predicted_structure: B x L x (num_atoms_per_residue * 3) tensor wrapped in a Coordinates object
Nz(Number of recycles must not be negative.c           	      ó�   <€ VSP                  W#R 7      ,           pSP                   F  pV! WW2SP                  R7      w  rK  	  W3# )r
  )r&  r1   r  )rí  rî  r  )rÕ   rN  Úresidxr&  Úblockr´   s   &&&& €r=   Ú
trunk_iterÚ+EsmFoldingTrunk.forward.<locals>.trunk_iter^  sF   ø€ Ø�D×6Ñ6°vÐ6ÓIÕIˆAàŸœ�Ù˜Q¨ÈtÏÉÔ_‘�‘1ñ %à�4ˆKr@   rd   rc   r¸  rT  r&   g      @g     `5@r(   r)   r„   )r)  r)  :Nrº  N)rd   r@  Úmax_recyclesr³   r9   Ú
zeros_likerÂ   rq   Úint64ru   r   r²   rð  ró  rÌ   rñ  rò  rõ  rö  r÷  ra   rê  Ú	distogramrï  )r´   Ú	seq_featsÚ
pair_featsÚtrue_aarþ  r&  Úno_recyclesrd   Ús_s_0Ús_z_0r   r(   r)   Ú	recycle_sÚ	recycle_zrï  Úrecycle_idxÚ	structures   f&&&&&&           r=   rÎ   ÚEsmFoldingTrunk.forwardI  s  ø€ ð ×!Ñ!ˆØˆØˆàÒØŸ+™+×2Ñ2‰Kà˜QŒÜ Ð!KÓLÐLØ˜1ÕˆKõ	ð ˆØˆÜ×$Ò$ SÓ)ˆ	Ü×$Ò$ SÓ)ˆ	Ü—{’{ C§I¡I¨c¨r NÐU¸6ÐUÌÏÉÑUˆä  ×-ˆKÜ  {°kÀAµoÔ'E¡ÌEÏMÊMËOÐK\×]Õ]à ×/Ñ/°	×0@Ñ0@Ó0BÓC×FÑFÀvÓN�	Ø ×/Ñ/°	×0@Ñ0@Ó0BÓC×FÑFÀvÓN�	Ø˜T×/Ñ/°×0CÑ0CÓ0EÓF×IÑIÈ&ÓQÕQ�	á% eÕ&7¸Õ9JÈFÓY‘�ð !×1Ñ1Ø˜tŸ™¨sÓ3°V¸T¿_¹_ÈSÓ=QÐRØØ—J‘J“Ló�	ð  �	Ø�	ä.×8Ñ8Ø˜kÕ*¨2Õ.¨xÕ8ØØØ×%Ñ%ó	 �÷% ^Ò]ñ .ð4 ˆ	�%ÑØˆ	�%ÑàÐ÷9 ^×]Ñ]ús   Ã7D(H8È8I
c                ó   € \         P                  ! VVV^,
          V P                  R7      pV^,          pV P                  ^RR7       Uu. uF  qUP	                  R4      NK  	  upw  rgpWv,
          p	W‡,
          p
V	P                  V
RR7      pR	V,          RV	,          ,           RV
,          ,
          V,           pVR
,          VR,          ,
          P                  ^4      P                  RRR7      p\         P                  ! WÔ8„  RR7      pV# u upi )r‹   ©rd   rÓ   g‡O[ŸI-â?g»:Ïñ4Má?T)rm   Úkeepdimsrð   r„   gÆ vlÂ¥â¿r~  ).r)  Nr)  )r9   r  rd   rÆ  r  ÚcrossÚpowr	  )ÚcoordsÚmin_binÚmax_binÚnum_binsÚ
boundariesrv   ÚNÚCAÚCr  r\  rC  ÚCBÚdistsr  s   &&&&           r=   r  ÚEsmFoldingTrunk.distogramŠ  sì   € ô —^’^ØØØ�q�LØ—=‘=ô	
ˆ
ð   •]ˆ
Ø+1¯<©<¸¸r¨<Ô+BÓCÑ+B a—I‘I˜b–MÑ+BÑC‰ˆˆqà�FˆØ�FˆØ�G‰G�A˜2ˆGÓˆØ˜1�_˜z¨A�~Õ-°
¸QµÕ>ÀÕCˆØ�OÕ$ r¨/Õ':Õ:×?Ñ?ÀÓB×FÑFÈ2ÐX\ÐFÓ]ˆÜ�yŠy˜Ñ+°Ô4ˆØˆùò Ds   Á
D)rî  r  r@  rí  rï  rò  rð  rñ  rõ  rö  r÷  )rB   rC   rD   rE   r±   rú  rÎ   Ústaticmethodr  rH   rI   r·   r¸   s   @@r=   rê  rê  *  s.   ù‡ € õ,ò.%ò?ðB ñó ÷ð r@   rê  a}  
    ESMForProteinFolding is the HuggingFace port of the original ESMFold model. It consists of an ESM-2 "stem" followed
    by a protein folding "head", although unlike most other output heads, this "head" is similar in size and runtime to
    the rest of the model combined! It outputs a dictionary containing predicted structural information about the input
    protein(s).
    c                   ó.  a a€ ] tR tRt oRR.tRtRtRtRtV 3R lt	V 3R lt
]V3R lR	 l4       t]RV3R
 lR ll4       tR tV3R lR ltR t]P&                  ! 4       RV3R lR ll4       t]V3R lR l4       tV3R lR ltV3R lR ltRtVtV ;t# )ÚEsmForProteinFoldingi£  r¬  r¦  FNc                óÞ   <€ \         SV `  V4       \        V\        4      '       dG   \        P
                  ! VP                  VP                  VP                  P                  4      4       R # R # rL   )
r°   r²  r“   r#  r£   rw  Ú
af2_to_esmÚ_af2_to_esm_from_vocab_listr@  Ú
vocab_list)r´   r³  rµ   s   &&€r=   r²  Ú"EsmForProteinFolding._init_weights³  sL   ø€ Ü‰Ñ˜fÔ%Ü�fÔ2×3Ò3Ü�JŠJ�v×(Ñ(¨&×*LÑ*LÈVÏ]É]×MeÑMeÓ*fÖgñ 4r@   c           
     ó*
  <€ \         SV `  V4       Wn        ^@V n        \	        VRR7      V n        V P
                  P                  R4       V P                  P                  P                  '       d   V P
                  P                  4        V P                  P                  V n        V P                  P                  V P                  P                  ,          V n        V P                  P                  V n        \         P"                  ! V P%                  VP&                  4      4      V n        \         P*                  ! \,        P.                  ! V P                  ^,           4      4      V n        V P                  P                  P2                  pVP4                  pVP6                  p\         P8                  ! \;        V P                  4      \         P<                  ! V P                  V4      \         P>                  ! 4       \         P<                  ! W34      4      V n         \B        PD                  ^,           V n#        ^ V n$        V PF                  ^,
          V n%        V PF                  ^,
          V n&        V P                  P&                  PO                  R4      V n(        V P                  P&                  PO                  R4      V n)        V P                  P&                  PO                  R4      V n*        V P                  P&                  PO                  R4      V n+        V P                  P                  PX                  '       d)   \         PZ                  ! V PF                  V^ R7      V n.        \_        V4      V n        \         P<                  ! W@P                  4      V n0        \         P<                  ! W@P                  4      V n1        \         P<                  ! W0PF                  4      V n2        ^2V n3        VPh                  p\         P8                  ! \         P:                  ! VPj                  4      \         P<                  ! VPj                  V P                  P                  Pl                  4      \         P<                  ! V P                  P                  Pl                  V P                  P                  Pl                  4      \         P<                  ! V P                  P                  Pl                  ^%V Pf                  ,          4      4      V n7        V Pq                  4        R# )	é@   F)Úadd_pooling_layerz<cls>z<mask>z<eos>ú<pad>)Úpadding_idxN)9r°   r±   r@  Údistogram_binsr   ÚesmÚrequires_grad_Úesmfold_configÚfp16_esmÚhalfÚhidden_sizeÚ	esm_featsÚnum_hidden_layersÚnum_attention_headsÚ	esm_attnsÚ
esm_layersr¾   rÛ  r&  r'  r%  r¿   r9   rÂ   Úesm_s_combineÚtrunkrà  r7  rô  r   r¿  rõ  Ú	esm_s_mlpr   Úrestype_numÚn_tokens_embedÚpad_idxÚunk_idxÚmask_idxÚindexÚesm_dict_cls_idxÚesm_dict_mask_idxÚesm_dict_eos_idxÚesm_dict_padding_idxÚembed_aar4  r5  rê  Údistogram_headÚptm_headÚlm_headÚ	lddt_binsrõ  rL  Úlddt_head_hid_dimr2   Ú	post_init)r´   r@  Útrunk_configrp  rq  Ústructure_module_configrµ   s   &&    €r=   r±   ÚEsmForProteinFolding.__init__¸  sD  ø€ Ü‰Ñ˜Ô àŒà ˆÔä˜F°eÔ<ˆŒà�‰×Ñ Ô&Ø�;‰;×%Ñ%×.×.Ð.Ø�H‰H�M‰MŒOàŸ™×0Ñ0ˆŒØŸ™×6Ñ6¸¿¹×9XÑ9XÕXˆŒØŸ+™+×7Ñ7ˆŒÜŸ)š) D×$DÑ$DÀV×EVÑEVÓ$WÓXˆŒÜŸ\š\¬%¯+ª+°d·o±oÈÕ6IÓ*JÓKˆÔà—{‘{×1Ñ1×7Ñ7ˆØ×-Ñ-ˆØ×-Ñ-ˆÜŸšÜ�d—n‘nÓ%Ü�IŠI�d—n‘n cÓ*Ü�GŠG‹IÜ�IŠI�cÓó	
ˆŒô 0×;Ñ;¸aÕ?ˆÔØˆŒØ×*Ñ*¨QÕ.ˆŒØ×+Ñ+¨aÕ/ˆŒØ $§¡× 6Ñ 6× <Ñ <¸WÓ EˆÔØ!%§¡×!7Ñ!7×!=Ñ!=¸hÓ!GˆÔØ $§¡× 6Ñ 6× <Ñ <¸WÓ EˆÔØ$(§K¡K×$:Ñ$:×$@Ñ$@ÀÓ$IˆÔ!Ø�;‰;×%Ñ%×.×.Ð.ÜŸ\š\¨$×*=Ñ*=¸sÐPQÔRˆDŒNä$ \Ó2ˆŒ
ä Ÿiši¨×-@Ñ-@ÓAˆÔÜŸ	š	 #×':Ñ':Ó;ˆŒÜ—y’y ×&9Ñ&9Ó:ˆŒØˆŒØ".×"?Ñ"?ÐÜŸšÜ�LŠLÐ0×=Ñ=Ó>Ü�IŠIÐ-×:Ñ:¸D¿K¹K×<VÑ<V×<hÑ<hÓiÜ�IŠI�d—k‘k×0Ñ0×BÑBÀDÇKÁK×D^ÑD^×DpÑDpÓqÜ�IŠI�d—k‘k×0Ñ0×BÑBÀBÈÏÉÕDWÓXó	
ˆŒð 	�‰Ör@   c                óJ   <€ V ^8„  d   QhRS[ S[,          RS[P                  /# )r!   r'  r^   )r_   r¦   r9   r`   )r;   r<   s   "€r=   r>   Ú!EsmForProteinFolding.__annotate__ò  s#   ø€ ÷ )ñ )±±Sµ	ð )¹e¿l¹lñ )r@   c                óÂ   € V P                  R 4      .\        P                   Uu. uF  qP                  V4      NK  	  up,           p\        P                  ! V4      # u upi )r,  )rB  r   Úrestypes_with_xr9   rˆ   )r'  rš   Úesm_reorders   &  r=   r&  Ú0EsmForProteinFolding._af2_to_esm_from_vocab_listñ  sR   € ð "×'Ñ'¨Ó0Ð1ÔRc×RsÒRsÓ4tÑRsÈQ×5EÑ5EÀaÖ5HÑRsÑ4tÕtˆÜ�|Š|˜KÓ(Ð(ùò 5us   ¤Ac                óÚ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[/# )	r!   Ú	input_idsÚattention_maskNÚposition_idsÚmasking_patternÚnum_recyclesÚoutput_hidden_statesr^   )r9   r`   r�   r¥   r   )r;   r<   s   "€r=   r>   rR  ø  s�   ø€ ÷ D7ñ D7á—<‘<ðD7ñ Ÿ™ tÕ+ðD7ñ —l‘l TÕ)ð	D7ñ
 Ÿ™¨Õ,ðD7ñ ˜D•jðD7ñ # T�kðD7ñ 
$ñD7r@   c           	     óJ  € V P                   P                  pTp	V	P                  ^ ,          p
V	P                  ^,          pVP                  pVf   \        P
                  ! WœR7      pVf'   \        P                  ! W¼R7      P                  V4      pV P                  W’4      pVe   V P                  W�W$4      w  rípMT	pRpV P                  V4      pVP                  V P                  P                  4      pVP                  '       d
   V^ ,          pVP                  4       pV P                  P!                  ^ 4      P#                  ^ 4      V,          P%                  ^4      pV P'                  V4      pVP)                  W«W¸P*                  P,                  4      pV P                   P                  P.                  '       d   VV P1                  V4      ,          pV P+                  VVW“W%R7      pVP3                  4        UUu/ uF  w  ppVR9   g   K  VVbK  	  pppV'       d   VVR&   V P5                  VR,          4      pVVP7                  ^^4      ,           ^,          pVVR&   V P9                  VR,          4      pVVR	&   V	VR
&   \;        V4       R F'  pVV;;,          VP#                  R4      ,          uu&   K)  	  VVR&   V P=                  VR,          4      P?                  VR,          P                  ^ ,          W«RV P@                  4      pVVR&   \C        VR,          V P@                  R7      pVVR&   V PE                  VR,          4      pVVR&   \G        V^V PH                  R7      VR&   VPK                  \M        V^V PH                  R7      4       \O        R/ VB # u uppi )aÍ  
masking_pattern (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Locations of tokens to mask during training as a form of regularization. Mask values selected in `[0, 1]`.
num_recycles (`int`, *optional*, defaults to `None`):
    Number of times to recycle the input sequence. If `None`, defaults to `config.num_recycles`. "Recycling"
    consists of passing the output of the folding trunk back in as input to the trunk. During training, the
    number of recycles should vary with each batch, to ensure that the model learns to output valid predictions
    after each recycle. During inference, num_recycles should be set to the highest value that the model was
    trained with for maximum accuracy. Accordingly, when this value is set to `None`, config.max_recycles is
    used.

Example:

```python
>>> from transformers import AutoTokenizer, EsmForProteinFolding

>>> model = EsmForProteinFolding.from_pretrained("facebook/esmfold_v1")
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/esmfold_v1")
>>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt", add_special_tokens=False)  # A tiny random peptide
>>> outputs = model(**inputs)
>>> folded_positions = outputs.positions
```

Nr  )r	  r)   r(   r'   Úmlm_targetsr*   r+   r,   r1   r2   r)  r3   r4   )r  Úno_binsr5   )r)   r(   r"   r#   r$   r%   r&   r'   )r-   r0   r„   rA   )(r@  r1  rq   rd   r9   Ú	ones_likeÚarangeÚ	expand_asÚaf2_idx_to_esm_idxÚ	bert_maskÚ&compute_language_model_representationsrÌ   r:  rc   Úesm_ablate_sequenceró  rÔ   r-  r  r<  rY  r;  r7  rG  r5  r’   rH  ró   rJ  r   r2   r…   rK  r*  rI  r   r.  rš  r   r   )r´   rX  rY  rZ  r[  r\  r]  ÚkwargsÚcfgÚaaÚBÚLrd   ÚesmaaÚ	masked_aar_  Úesm_sr
  r  r  ri   rš   Údisto_logitsr+   r2   r3   r4   s   &&&&&&&,                   r=   rÎ   ÚEsmForProteinFolding.forward÷  sL  € ðF �k‰k×(Ñ(ˆàˆØ�H‰H�Q�KˆØ�H‰H�Q�KˆØ×!Ñ!ˆØÒ!Ü"Ÿ_š_¨RÔ?ˆNØÒÜ Ÿ<š<¨Ô9×CÑCÀIÓNˆLð ×'Ñ'¨Ó;ˆàÒ&Ø,0¯N©N¸2ÀnÓ,fÑ)ˆI™kàˆIØˆKð ×;Ñ;¸EÓBˆð
 —‘˜×+Ñ+×1Ñ1Ó2ˆà×"×"Ð"Ø˜A•IˆEà—‘“ˆð ×#Ñ#×+Ñ+¨AÓ.×8Ñ8¸Ó;¸eÕC×LÑLÈQÓOˆØ—‘˜uÓ%ˆà—‘  a¯©×)EÑ)EÓFˆà�;‰;×%Ñ%×.×.Ð.Ø�T—^‘^ IÓ.Õ.ˆEàŸ*™* U¨E°2À^˜*Ónˆ	ð "Ÿ™Ô)ô
á)‘��1Øð	ñ
ô ˆAˆqŠDÙ)ð 	ñ 
÷" Ø'2ˆI�mÑ$à×*Ñ*¨9°UÕ+;Ó<ˆØ$ |×'=Ñ'=¸aÀÓ'CÕCÀqÕHˆØ(4ˆ	Ð$Ñ%à—L‘L ¨5Õ!1Ó2ˆ	Ø!*ˆ	�+Ñà ˆ	�(ÑÜ˜)Ô$ó

ˆAð �a�L˜N×4Ñ4°RÓ8Õ8�Lñ	
ð
 &2ˆ	�/Ñ"à—N‘N 9¨XÕ#6Ó7×?Ñ?À	È(Õ@S×@YÑ@YÐZ[Õ@\Ð^_ÐdfÐhl×hvÑhvÓwˆ	Ø!*ˆ	�+ÑÜ  ¨2¥°T·^±^ÔDˆØ"ˆ	�'Ñà—]‘] 9¨UÕ#3Ó4ˆ
Ø",ˆ	�,ÑÜ% j¸"Àd×FYÑFYÔZˆ	�%ÑØ×ÑÔ8¸ÈRÐY]×YlÑYlÔmÔnä)Ñ6¨IÑ6Ð6ùóe
s   Ç=NÈNc                óþ   € V P                   P                  VP                  8w  d+   V P                   P                  VP                  4      V n         V^,           P                  V^8g  ^ 4      pV P                   V,          # r}  )r%  rd   rÌ   rÈ  )r´   rj  r&  s   &&&r=   rd  Ú'EsmForProteinFolding.af2_idx_to_esm_idx~  s[   € à�?‰?×!Ñ! R§Y¡YÔ.Ø"Ÿo™o×0Ñ0°·±Ó;ˆDŒOØ�1�f×!Ñ! $¨!¡)¨QÓ/ˆØ�‰˜rÕ"Ð"r@   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r!   rm  r^   rû   )r;   r<   s   "€r=   r>   rR  …  s#   ø€ ÷ ñ ¹E¿L¹Lð ÉUÏ\É\ñ r@   c           	     óÄ  € \        V P                  4       4      P                  pVP                  w  r4V P                  P
                  P                  '       dC   \        P                  ! W4V P                  P                  ^ ,          RV P                  VR7      pV# V P                  V P                  rvVP                  V^3V4      pVP                  V^3V P                  4      p	\        P                   ! W�V	.^R7      pWq\#        V4      V^8g  P%                  ^4      3&   V P'                  W^8g  RR7      R,          p
\        P(                  ! V
^R7      pVR^R13,          pV# )r   r  rÓ   T)rY  r]  Úhidden_statesr)  r„   )ÚnextÚ
parametersrd   rq   r@  r1  Ú	bypass_lmr9   rÂ   r:  Úsizer5  rC  rE  Únew_fullrF  rä  ru   r	  r/  r€  )r´   rm  rd   rk  rl  ro  ÚbosiÚeosiÚbosÚeosÚesm_hidden_statess   &&         r=   rf  Ú;EsmForProteinFolding.compute_language_model_representations…  s(  € Ü�d—o‘oÓ'Ó(×/Ñ/ˆØ�{‰{‰ˆà�;‰;×%Ñ%×/×/Ð/Ü—K’K  d×&8Ñ&8×&=Ñ&=¸aÕ&@À"ÀdÇnÁnÐ]cÔdˆEØˆLà×*Ñ*¨D×,AÑ,AˆdØ�n‰n˜a ˜V TÓ*ˆØ�n‰n˜a ˜V T×%>Ñ%>Ó?ˆÜ—	’	˜3 sÐ+°Ô3ˆà/3Œe�A‹h˜ !™×(Ñ(¨Ó+Ð+Ñ,ð
 !ŸH™H UÀA¹:Ð\`˜HÓaÐbqÕrÐÜ—’Ð-°1Ô5ˆà�a˜˜2˜�g•ˆàˆr@   c                ó¼   € VP                  4       pVP                  4       pVP                  4       pV P                  WT^8H  &   ^ Wd^8g  &   V P                  Wt^8H  &   WWV3# r}  )rx  rA  rD  )r´   rj  rm  r&  ÚpatternÚnew_aaÚtargetÚ	new_esmaas   &&&&&   r=   re  ÚEsmForProteinFolding.bert_maskž  s\   € Ø—‘“ˆØ—‘“ˆØ—K‘K“Mˆ	Ø#Ÿ}™}ˆ˜!‰|ÑØ ˆ˜!‰|ÑØ"&×"8Ñ"8ˆ	˜Q‘,ÑØ &Ð(Ð(r@   c                ó@   <€ V ^8„  d   QhRS[ S[S[ ,          ,          /# )r!   Úseqs)r¦   r_   )r;   r<   s   "€r=   r>   rR  ¨  s   ø€ ÷ %
ñ %
á‘D™•I�oñ%
r@   c                óø  € \        V\        4      '       d   V.pMTp\        V P                  4       4      P                  p\        V Uu. uF]  p\        P                  ! \        P                  ! V\        P                  R R7      4      P                  V4      P                  ^R7      NK_  	  up4      p\        V Uu. uF  qVP                  \        V4      4      NK  	  up4      pVfC   \        P                  ! VP                   ^,          VR7      P#                  \        V4      R4      MVP                  V4      pVP$                  ^8X  d   VP'                  ^ 4      pV P)                  VVVR7      # u upi u upi )T)ÚsequenceÚmappingÚmap_unknown_to_xrÓ   r  )rZ  r„   )r“   r¦   rw  rx  rd   r}   r9   Ú
from_numpyr   Úsequence_to_onehotÚrestype_order_with_xrÌ   Úargmaxr,  rl   rb  rq   r,  r  r-  rÎ   )r´   r‰  rZ  rw   rd   Úseqr,   r&  s   &&&     r=   ÚinferÚEsmForProteinFolding.infer§  sT  € ô �dœC× Ò Ø�&‰CàˆCä�d—o‘oÓ'Ó(×/Ñ/ˆÜ&ñ óñ �Cô × Ò Ü%×8Ò8Ø!$Ü 1× FÑ FØ)-ôó÷ ‘�F“ß‘˜A�“òñ ñó
ˆô %É3Ó%OÉ3ÀC§o¡o´c¸#³hÖ&?É3Ñ%OÓPˆð Ò#ô �LŠL˜Ÿ™ a�°Ô8×?Ñ?ÄÀCÃÈ"ÔMà—‘ Ó(ð 	ð
 ×Ñ Ô!Ø'×1Ñ1°!Ó4ˆLØ�|‰|ØØØ%ð ó 
ð 	
ùò+ùò &Ps   Á	A#E2Â<"E7c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# )r!   r  r^   )r”   r_   r¦   )r;   r<   s   "€r=   r>   rR  Ð  s   ø€ ÷ ñ ™dð ¡t©C¥yñ r@   c           	     ó  € V P                  4        UUu/ uF$  w  rWP                  R4      P                  4       bK&  	  p pp. p\        V R,          R,          V 4      pV R,          p\	        V R,          P
                  ^ ,          4       Fr  pV R,          V,          pWF,          pWV,          p	V R,          V,          ^,           p
\        VVV	V
V R,          V,          R7      pVP                  \        V4      4       Kt  	  V# u uppi )	zDReturns the pdb (file) string from the model given the model output.r“  r&   r0   r,   r1   r3   )r,   Úatom_positionsrØ  r1   Ú	b_factorsr„   )	r’   rÌ   Únumpyr   ru   rq   r   rR  r   )r  ri   rš   ÚpdbsÚfinal_atom_positionsÚfinal_atom_maskrz   rj  Úpred_posr&  ÚresidÚpreds   &           r=   Úoutput_to_pdbÚ"EsmForProteinFolding.output_to_pdbÏ  sî   € ð 6<·\±\´^ÔD±^©T¨Q�!—T‘T˜%“[×&Ñ&Ó(Ò(±^ˆÑDØˆÜ/°°{Õ0CÀBÕ0GÈÓPÐØ Ð!5Õ6ˆÜ�v˜hÕ'×-Ñ-¨aÕ0Ö1ˆAØ˜Õ! !Õ$ˆBØ+Õ.ˆHØ"Õ%ˆDØ˜?Õ+¨AÕ.°Õ2ˆEÜØØ'ØØ#Ø  �/¨!Õ,ôˆDð �K‰Kœ˜t›Ö%ñ 2ð ˆùó# Es   ”*Dc                ó    <€ V ^8„  d   QhRS[ /# )r!   r^   )r¦   )r;   r<   s   "€r=   r>   rR  å  s   ø€ ÷ -ñ -±#ñ -r@   c                óŽ   € \        V\        4      '       g   Q hV P                  ! V.VO5/ VB pV P                  V4      ^ ,          # ©zEReturns the pdb (file) string from the model given an input sequence.)r“   r¦   r“  r   ©r´   r‰  Úargsrh  r  s   &&*, r=   Ú	infer_pdbÚEsmForProteinFolding.infer_pdbå  sC   € ä˜$¤×$Ò$Ð$Ð$Ø—’˜DÐ2 4Ò2¨6Ñ2ˆØ×!Ñ! &Ó)¨!Õ,Ð,r@   c                óF   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          /# )r!   r‰  r^   )r_   r¦   )r;   r<   s   "€r=   r>   rR  ë  s#   ø€ ÷ *ñ *™t¡C�yð *¹dÁ3½iñ *r@   c                óP   € V P                   ! V.VO5/ VB pV P                  V4      # r¤  )r“  r   r¥  s   &&*, r=   Ú
infer_pdbsÚEsmForProteinFolding.infer_pdbsë  s+   € à—’˜DÐ2 4Ò2¨6Ñ2ˆØ×!Ñ! &Ó)Ð)r@   )r%  r@  r.  rH  r5  r/  r8  rC  rE  rD  rF  r5  r9  r:  r<  rK  r2   rJ  rA  r>  r?  rI  r;  r@  )NNNNFrL   )rB   rC   rD   rE   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_attention_backendÚ_can_record_outputsr²  r±   r!  r&  r   rÎ   rd  rf  re  r9   r²   r“  r   r§  r«  rH   rI   r·   r¸   s   @@r=   r#  r#  £  sÈ   ù‡ € ð 2Ð3XÐYÐØ ÐØ€NØ"'ÐàÐõhõ
7ðr ÷)ó ð)ð
 ÷D7ñ D7ó ðD7òL#÷ð ò2)ð ‡]‚]ƒ_÷%
ñ %
ó ð%
ðN ÷ó ð÷*-ð -÷*÷ *ð *r@   r#  r‘   rƒ   )r'  )Urô   r‚  Úcollections.abcr   r   Údataclassesr   Ú	functoolsr   r™  rÉ  r9   Útorch.nnr¾   r   Ú r   r£   Úintegrations.deepspeedr	   Úmodeling_outputsr
   rW   r   r   r   Úutils.genericr   Úmodeling_esmr   r   Úopenfold_utilsr   r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrB   Úloggerr   rS   rZ   r}   r†   r�   r•   r¿  rž   ÚModulerº   r2  r3  rÖ   rØ   r  r5  r˜  r¸  rÎ  rÝ  rê  rò  r¦  r  r*  r.  r0  r=  rJ  r£  r’  r�  r¤  r¬  rê  r#  Ú__all__rA   r@   r=   Ú<module>rÀ     s‹  ðó Û 
ß .Ý !Ý ã Û Ý Ý å &Ý <Ý +÷ñ õ
 ,ß 6÷÷ ÷ ó ð  
×	Ò	˜HÓ	%€ñ ðôð
 ôJA ó JAó óðJAòZò
÷õ.1õGò
ô+5�B—I‘Iô +5ô\�r—y‘yô ð* ‡�×Ñöó ðôU�r—y‘yô Uôpk˜rŸy™yô kô\G¨"¯)©)ô GôT
9*Ð/ô 9*ôx;(˜2Ÿ9™9ô ;(ô|3�R—Y‘Yô 3ô,#˜BŸI™Iô #ôL˜BŸI™Iô ô*˜Ÿ	™	ô ô e.¨"¯)©)ô e.÷PPñ Pô&;ò
ô,!˜bŸi™iô !ôH˜bŸi™iô ô(>!˜Ÿ™ô >!ôBC R§Y¡Yô CôL˜BŸI™Iô ô.¨B¯I©Iô ô. r§y¡yô ô.C
˜RŸY™Yô C
ôLr�b—i‘iô rñr ðôôC*Ð-ó C*óðC*ðL
 "Ð#;Ð
<‚r@   