+
    "Ý³j¥3  ã                   ó*  € ^ RI Ht ^ RIt^ RIHt ^ RIHt ^ RIHt ^RIH	t	 ^RI
Ht ^RIHt ^R	IHt  ! R
 R]4      tR R lt ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      tR# )é    )ÚOptionalN)ÚTensor)ÚDTensor)Ú	Optimizer)Ú_fp32_to_bf16_sr)ÚOptimState4bit)ÚOptimState8bit)ÚOptimStateFp8c                   óÂ   a a€ ] tR t^t oV3R lV 3R lltV3R lV 3R lltV 3R lt]V3R lR l4       tV3R lR	 lt	]
P                  ! 4       RR
 l4       tRtVtV ;t# )Ú	_AdamBasec                ó   <€ V ^8„  d   QhRR/# ©é   ÚreturnN© )ÚformatÚ__classdict__s   "€ÚS/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torchao/optim/adam.pyÚ__annotate__Ú_AdamBase.__annotate__   s   ø€ ÷ !ñ !ð 
ñ!ó    c               óÒ  <€ R V8:  g   \        RP                  V4      4      hR V8:  g   \        RP                  V4      4      hR V^ ,          u;8:  d   R8  g$   M \        RP                  V^ ,          4      4      hR V^,          u;8:  d   R8  g$   M \        RP                  V^,          4      4      h\        VVVVVR7      p
\        SV `  W4       Wpn        W€n        W�n        R# )ç        zInvalid learning rate: {}zInvalid epsilon value: {}g      ð?z%Invalid beta parameter at index 0: {}z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚepsÚweight_decayÚamsgradN)Ú
ValueErrorr   ÚdictÚsuperÚ__init__Ú
block_sizeÚbf16_stochastic_roundÚis_adamw)ÚselfÚparamsr   r   r   r   r   r#   r$   r%   ÚdefaultsÚ	__class__s   &&&&&&&$$$ €r   r"   Ú_AdamBase.__init__   sÓ   ø€ ð �bŒyÜÐ8×?Ñ?ÀÓCÓDÐDØ�cŒzÜÐ8×?Ñ?ÀÓDÓEÐEØ�e˜A•hÖ$ Ö$ÜÐD×KÑKÈEÐRSÍHÓUÓVÐVØ�e˜A•hÖ$ Ö$ÜÐD×KÑKÈEÐRSÍHÓUÓVÐVÜØØØØ%Øô
ˆô 	‰Ñ˜Ô*Ø$ŒØ%:Ô"Ø Žr   c                ó$   <€ V ^8„  d   QhRS[ RR/# )r   Úparam_groupr   N)r    )r   r   s   "€r   r   r   5   s   ø€ ÷ Iñ I©4ð I°Dñ Ir   c                óê   <€ \         SV `  V4       V P                  R,          p\        VR,          \        4      '       g3   \
        P                  ! VR,          \
        P                  R7      VR&   R# R# )é   r   ©ÚdtypeNéÿÿÿÿ)r!   Úadd_param_groupÚparam_groupsÚ
isinstancer   ÚtorchÚtensorÚfloat32)r&   r,   Úgroupr)   s   && €r   r2   Ú_AdamBase.add_param_group5   sT   ø€ Ü‰Ñ Ô,ð ×!Ñ! "Õ%ˆÜ˜% �+¤v×.Ò.ÜŸ,š, u¨T¥{¼%¿-¹-ÔHˆE�$‹Kñ /r   c                óp   <€ \         SV `  V4       V P                   F  pVP                  R R4       K  	  R# )r   FN)r!   Ú__setstate__r3   Ú
setdefault)r&   Ústater8   r)   s   && €r   r;   Ú_AdamBase.__setstate__=   s0   ø€ Ü‰Ñ˜UÔ#Ø×&Ô&ˆEØ×Ñ˜Y¨Ö.ó 'r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# ©r   ÚpÚsignedr#   ©r   ÚboolÚint)r   r   s   "€r   r   r   D   s"   ø€ ÷ "ñ "™6ð "©4ð "¹Sñ "r   c                ó   € \         h©N)ÚNotImplementedError©rA   rB   r#   s   &&&r   Ú_subclass_zerosÚ_AdamBase._subclass_zerosC   s   € ä!Ð!r   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rA   rB   )r   rD   )r   r   s   "€r   r   r   G   s   ø€ ÷ ñ ™Vð ©Tñ r   c           	     ó$  € \        V\        4      '       d   VP                  4       MTpVP                  4       R 8¼  dD   VP                  4       V P                  ,          ^ 8X  d   V P                  W2V P                  4      pM\        P                  ! V4      p\        V\        4      '       dI   \        P                  ! VVP                  VP                  RVP                  VP                  4       R7      pVP                  VP                  4      pV# )i   F)Úlocal_tensorÚdevice_meshÚ
placementsÚ	run_checkÚshapeÚstride)r4   r   Úto_localÚnumelr#   rJ   r5   Ú
zeros_likeÚ
from_localrO   rP   rR   rS   ÚtoÚdevice)r&   rA   rB   Úlocal_pÚouts   &&&  r   Ú_new_bufferÚ_AdamBase._new_bufferG   sÃ   € Ü",¨Q´×"8Ò"8�!—*‘*”,¸aˆð �=‰=‹?˜dÔ" w§}¡}£¸¿¹Õ'HÈAÔ'MØ×&Ñ& w¸¿¹ÓH‰Cä×"Ò" 7Ó+ˆCô �aœ×!Ò!Ü×$Ò$Ø ØŸM™MØŸ<™<ØØ—g‘gØ—x‘x“zôˆCð �f‰f�Q—X‘XÓˆØˆ
r   c                ó  € R pVe.   \         P                  ! 4       ;_uu_ 4        V! 4       pR R R 4       \         P                  P                  P	                  4       ;_uu_ 4        V P
                   EFá  pVR,           EFÏ  pVP                  f   K  VP                  pVP                  '       d   \        R4      hV P                  V,          p\        V4      ^ 8X  dh   \         P                  ! R4      VR&   V P                  VR4      VR&   V P                  VR4      VR&   VR	,          '       d   V P                  VR4      VR
&   VR;;,          ^,          uu&   \        VR,          \        4      '       g   \        R4      h\         P                  ! \         RRR7      ! VP#                  4       TVR,          VR,          VR,          VP%                  R
R 4      VR,          VR,          ^ ,          VR,          ^,          VR,          VR,          V P&                  V P(                  ;'       d    VP*                  \         P,                  J 4       EKÒ  	  EKä  	  R R R 4       V#   + '       g   i     EL>; i  + '       g   i     T# ; i)Nr'   z Sparse gradient is not supportedr   ÚstepTÚexp_avgFÚ
exp_avg_sqr   Úmax_exp_avg_sqr   zulr was changed to a non-Tensor object. If you want to update lr, please use optim.param_groups[0]['lr'].fill_(new_lr))Ú	fullgraphÚdynamicr   r   r   )r5   Úenable_gradÚ_dynamoÚutilsÚdisable_cache_limitr3   ÚgradÚ	is_sparseÚRuntimeErrorr=   Úlenr6   r\   r4   r   ÚcompileÚsingle_param_adamÚdetachÚgetr%   r$   r0   Úbfloat16)r&   ÚclosureÚlossr8   rA   ri   r=   s   &&     r   r_   Ú_AdamBase.stepe   s  € àˆØÒÜ×"Ò"×$Õ$Ù“y�÷ %ô
 �]‰]× Ñ ×4Ñ4×6Õ6Ø×*Õ*�Ø˜xŸ™�AØ—v‘v’~Ù àŸ6™6�DØ—~—~�~Ü*Ð+MÓNÐNà ŸJ™J q�M�Eô ˜5“z Q”Ü(-¯ª°SÓ(9˜˜f™Ø+/×+;Ñ+;¸A¸tÓ+D˜˜iÑ(Ø.2×.>Ñ.>¸qÀ%Ó.H˜˜lÑ+Ø  ×+Ô+Ø6:×6FÑ6FÀqÈ%Ó6P˜EÐ"2Ñ3à˜&—M QÕ&“Mä% e¨D¥k´6×:Ò:Ü*ðHóð ô —M’MÔ"3¸tÈUÕSØŸ™›
ØØ˜f�Ø˜iÕ(Ø˜lÕ+ØŸ	™	Ð"2°DÓ9Ø˜d�Ø˜g� qÕ)Ø˜g� qÕ)Ø˜nÕ-Ø˜e�ØŸ™Ø×2Ñ2×PÐP°q·w±wÄ%Ç.Á.Ð7P÷ô= )ñ +÷ 7ð` ˆ÷k %×$Ð$ú÷
 7Ö6ð` ˆús,   ¤I!Á$AI5Â0A?I5Ä0C;I5È,+I5É!I2	É5J	)r$   r#   r%   rG   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   r2   r;   ÚstaticmethodrJ   r\   r5   Úno_gradr_   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r)   r   s   @@r   r   r      s^   ù‡ € ÷!ó !÷BIó Iõ/ð ÷"ó ð"÷ð ð< ‡]‚]ƒ_ó8ó ÷8ð 8r   r   c                óÊ   € V ^8„  d   QhR\         R\         R\         R\         R\         R\        \         ,          R\         R\        R	\        R
\        R\        R\        R\        /# )r   rA   ri   r_   r`   ra   rb   r   Úbeta1Úbeta2r   r   ÚIS_ADAMWÚBF16_STOCHASTIC_ROUND)r   r   ÚfloatrD   )r   s   "r   r   r   £   s�   € ÷ .ñ .Üð.ä
ð.ô ð.ô ð	.ô
 ð.ô œVÕ$ð.ô 	ð.ô ð.ô ð.ô ð.ô 
ð.ô ð.ô  ñ.r   c                 ó¤  € V P                  4       pVP                  4       pV'       d   WÖV	,          V,          ,
          pMWéV,          ,           p^Wr,          ,
          p^W‚,          ,
          pVP                  4       P                  V^V,
          4      pVP                  4       P                  VP                  4       ^V,
          4      pVP                  V4       VP                  V4       Ved   \        P
                  ! VP                  4       V4      pVP                  V4       VP                  4       VP                  4       ,          V
,           pM,VP                  4       VP                  4       ,          V
,           pWÖVV,          ,          V,          ,
          pV'       d   V P                  \        V4      4       R# V P                  V4       R# )r.   N)r„   ÚlerpÚsquareÚcopy_r5   ÚmaximumÚsqrtr   )rA   ri   r_   r`   ra   rb   r   r€   r�   r   r   r‚   rƒ   Úp_f32Úgrad_f32Úbias_correction1Úbias_correction2Úexp_avg_f32Úexp_avg_sq_f32Úmax_exp_avg_sq_f32Údenoms   &&&&&&&&&&&&&        r   rn   rn   £   s]  € ð  �G‰G‹I€EØ�z‰z‹|€HçØ˜\Õ)¨EÕ1Õ1‰à¨UÕ2Õ2ˆà˜5�;•ÐØ˜5�;•Ðð —-‘-“/×&Ñ& x°°UµÓ;€KØ×%Ñ%Ó'×,Ñ,¨X¯_©_Ó->ÀÀEÅ	ÓJ€Nà‡M�M�+ÔØ×Ñ�^Ô$àÒ!Ü"Ÿ]š]¨>×+?Ñ+?Ó+AÀ>ÓRÐØ×ÑÐ/Ô0Ø#×(Ñ(Ó*Ð-=×-BÑ-BÓ-DÕDÈÕK‰à×$Ñ$Ó&Ð)9×)>Ñ)>Ó)@Õ@ÀCÕGˆà˜+Ð(8Õ8Õ9¸EÕAÕA€EçØ	�‰Ô  Ó'Ö(à	�‰�Žr   c                   ój   a a€ ] tR t^Ôt oR
RRRR/V3R lV 3R llllt]V3R lR l4       tR	tVtV ;t	# )ÚAdam8bitFr#   é   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdam8bit.__annotate__Õ   ó   ø€ ÷ ?ñ ?ð 
ñ?r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )F©r#   r$   r%   ztorchao.optim.Adam8bitN©r!   r"   r5   Ú_CÚ_log_api_usage_once©
r&   r'   r   r   r   r   r   r#   r$   r)   s
   &&&&&&&$$€r   r"   ÚAdam8bit.__init__Õ   óG   ø€ ô 	‰ÑØØØØØØØ!Ø"7Øð 	ô 
	
ô 	�‰×$Ñ$Ð%=Ö>r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   r—   ï   ó"   ø€ ÷ 
ñ 
™6ð 
©4ð 
¹Sñ 
r   c                ór   € \         P                  ! V P                  WV P                  V P                  R 7      # ©r/   ©r	   ÚzerosrR   rY   r0   rI   s   &&&r   rJ   ÚAdam8bit._subclass_zerosî   ó*   € ä×#Ò#Ø�G‰G�V¨¯©¸¿¹ô
ð 	
r   r   ©çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   F©
ru   rv   rw   rx   r"   ry   rJ   r{   r|   r}   r~   s   @@r   r”   r”   Ô   ó<   ù‡ € ñ?ð ð?ð $÷?ö ?ð2 ÷
ó ÷
ð 
r   r”   c                   ój   a a€ ] tR t^õt oR	R^€RR/V3R lV 3R llllt]V3R lR l4       tRtVtV ;t	# )
ÚAdam4bitFr#   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdam4bit.__annotate__ö   r˜   r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )Frš   ztorchao.optim.Adam4bitNr›   rž   s
   &&&&&&&$$€r   r"   ÚAdam4bit.__init__ö   r    r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   r²     r¢   r   c                ór   € \         P                  ! V P                  WV P                  V P                  R 7      # r¤   ©r   r¦   rR   rY   r0   rI   s   &&&r   rJ   ÚAdam4bit._subclass_zeros  r¨   r   r   r©   r­   r~   s   @@r   r°   r°   õ   r®   r   r°   c                   ój   a a€ ] tR tRt oRRRRR/V3R lV 3R llllt]V3R lR	 l4       tR
tVtV ;t	# )ÚAdamFp8i  Fr#   r•   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdamFp8.__annotate__  s   ø€ ÷ >ñ >ð 
ñ>r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )Frš   ztorchao.optim.AdamFp8Nr›   rž   s
   &&&&&&&$$€r   r"   ÚAdamFp8.__init__  sG   ø€ ô 	‰ÑØØØØØØØ!Ø"7Øð 	ô 
	
ô 	�‰×$Ñ$Ð%<Ö=r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   r¼   1  ó'   ø€ ÷ Qñ Q™6ð Q©4ð Q¹Sñ Qr   c                óp   € \         P                  ! V P                  W P                  V P                  R 7      # r¤   ©r
   r¦   rR   rY   r0   rI   s   &&&r   rJ   ÚAdamFp8._subclass_zeros0  ó#   € ä×"Ò" 1§7¡7¨J¿¹ÈÏÉÔPÐPr   r   r©   r­   r~   s   @@r   rº   rº     s?   ù‡ € ñ>ð ð>ð $÷>ö >ð2 ÷Qó ÷Qð Qr   rº   c                   ój   a a€ ] tR tRt oRRRRR/V3R lV 3R llllt]V3R lR	 l4       tR
tVtV ;t	# )Ú	AdamW8biti5  Fr#   r•   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdamW8bit.__annotate__6  ó   ø€ ÷ @ñ @ð 
ñ@r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )Trš   ztorchao.optim.AdamW8bitNr›   rž   s
   &&&&&&&$$€r   r"   ÚAdamW8bit.__init__6  óG   ø€ ô 	‰ÑØØØØØØØ!Ø"7Øð 	ô 
	
ô 	�‰×$Ñ$Ð%>Ö?r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   rÈ   P  r¢   r   c                ór   € \         P                  ! V P                  WV P                  V P                  R 7      # r¤   r¥   rI   s   &&&r   rJ   ÚAdamW8bit._subclass_zerosO  r¨   r   r   ©rª   r«   r¬   g{®Gáz„?Fr­   r~   s   @@r   rÆ   rÆ   5  ó@   ù‡ € ñ@ð ð@ð $÷@ö @ð2 ÷
ó ÷
ð 
r   rÆ   c                   ój   a a€ ] tR tRt oR
R^€RR/V3R lV 3R llllt]V3R lR l4       tR	tVtV ;t	# )Ú	AdamW4bitiV  Fr#   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdamW4bit.__annotate__W  rÉ   r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )Trš   ztorchao.optim.AdamW4bitNr›   rž   s
   &&&&&&&$$€r   r"   ÚAdamW4bit.__init__W  rÌ   r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   rÕ   q  r¢   r   c                ór   € \         P                  ! V P                  WV P                  V P                  R 7      # r¤   r·   rI   s   &&&r   rJ   ÚAdamW4bit._subclass_zerosp  r¨   r   r   rÐ   r­   r~   s   @@r   rÓ   rÓ   V  rÑ   r   rÓ   c                   ój   a a€ ] tR tRt oRRRRR/V3R lV 3R llllt]V3R lR	 l4       tR
tVtV ;t	# )ÚAdamWFp8iw  Fr#   r•   r$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   ÚAdamWFp8.__annotate__x  r˜   r   c               óv   <€ \         S	V `  VVVVVVVVR R7	       \        P                  P	                  R4       R# )Trš   ztorchao.optim.AdamWFp8Nr›   rž   s
   &&&&&&&$$€r   r"   ÚAdamWFp8.__init__x  sG   ø€ ô 	‰ÑØØØØØØØ!Ø"7Øð 	ô 
	
ô 	�‰×$Ñ$Ð%=Ö>r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r@   rC   )r   r   s   "€r   r   rÞ   ’  rÀ   r   c                óp   € \         P                  ! V P                  W P                  V P                  R 7      # r¤   rÂ   rI   s   &&&r   rJ   ÚAdamWFp8._subclass_zeros‘  rÄ   r   r   rÐ   r­   r~   s   @@r   rÜ   rÜ   w  s?   ù‡ € ñ?ð ð?ð $÷?ö ?ð2 ÷Qó ÷Qð Qr   rÜ   c                   óJ   a a€ ] tR tRt oRRR/V3R lV 3R lllltRtVtV ;t# )Ú_AdamWi–  Fr$   c                ó   <€ V ^8„  d   QhRR/# r   r   )r   r   s   "€r   r   Ú_AdamW.__annotate__—  s   ø€ ÷ 
ñ 
ð 
ñ
r   c               óJ   <€ \         SV `  VVVVVV\        R4      VRR7	       R# )z�AdamW optimizer that supports quantized training (parameter is quantized). This optimizer should
only be used with torchao's quantized training.ÚinfTrš   N)r!   r"   r„   )	r&   r'   r   r   r   r   r   r$   r)   s	   &&&&&&&$€r   r"   Ú_AdamW.__init__—  s6   ø€ ô 	‰ÑØØØØØØÜ˜U“|Ø"7Øð 	ö 
	
r   r   rÐ   )ru   rv   rw   rx   r"   r{   r|   r}   r~   s   @@r   rå   rå   –  s   ù‡ € ñ
ð $÷
÷ 
ö 
r   rå   )Útypingr   r5   r   Útorch.distributed._tensorr   Útorch.optimr   Úquant_utilsr   Úsubclass_4bitr   Úsubclass_8bitr	   Úsubclass_fp8r
   r   rn   r”   r°   rº   rÆ   rÓ   rÜ   rå   r   r   r   Ú<module>rò      s“   ðõ ã Ý Ý -Ý !å )Ý )Ý )Ý 'ôK�	ô Kõ`.ôb
ˆyô 
ôB
ˆyô 
ôBQˆiô Qô>
�	ô 
ôB
�	ô 
ôBQˆyô Qô>
ˆYö 
r   