+
    "Ý³j;  ã                   óL   € ^ RI Ht ^ RIt^ RIHtHt ^ RIHt  ! R R]4      tR# )é    )ÚTypeN)Ú	OptimizerÚParamsT)Úget_available_devicesc                   óÄ   a € ] tR t^t o ]P
                  P                  3RRRR/V 3R lR lllt]P                  ! 4       RR l4       t	RR	 lt
]R
 4       tR tR tRtV tR# )ÚCPUOffloadOptimizerÚoffload_gradientsFÚminimal_sizei   c          
      óF   <€ V ^8„  d   QhRS[ RS[S[,          RS[RS[RR/# )é   ÚparamsÚoptimizer_classr	   r
   ÚreturnN)r   r   r   ÚboolÚint)ÚformatÚ__classdict__s   "€ÚZ/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torchao/optim/cpu_offload.pyÚ__annotate__Ú CPUOffloadOptimizer.__annotate__   sO   ø€ ÷ _Hñ _Háð_Hñ ™i�ð_Hñ
  ð_Hñ ð_Hð 
ñ_Hó    c               ó  a a€ V\         P                  P                  J d   RV9  d   VP                  RR7       \	        V4      p\        V4      ^ 8X  d   \        R4      h\        V^ ,          \        4      '       g   RV/.pVS n	        RS n
        . S n        \        4       S n        \        4       S n        \        4       R,          S n        S P                  R9   g   Q R4       h\!        \         S P                  4      P#                  4       S n        \        4       S n        VV 3R lpV EF*  pVP)                  R4      p. p	V FÛ  p
V
P*                  '       g   K  V
P-                  4       S P                  8  d   V	P/                  V
4       KI  \         P0                  ! V
R	RR
7      p\         P0                  ! VRR7      Vn        VP5                  V
P7                  4       RR7       VS P                  V
&   V
P9                  V4       V! RV/VC.3/ VB S P                  V
&   KÝ  	  \        V	4      ^ 8”  g   EK  S P                  P/                  RV	/VC4       EK-  	  \        S P                  4      ^ 8”  d   V! S P                  3/ VB S n
        R# R# )am  Offload optimizer to CPU for single-GPU training. This will reduce GPU memory by the size of optimizer state.
Optimizer step will be done on CPU.

Args
    params: a list of parameters or parameter groups.
    optimizer_class: constructor of the base optimizer. Defaults to :class:`torch.optim.AdamW`.
    offload_gradients: free GPU gradients once they are moved to CPU. Not compatible with gradient accumulation.
    minimal_size: tensors smaller than this are kept on the GPU, to avoid excessively many small transfers.
    kwargs: other keyword arguments to be passed to the base optimizer e.g. `lr`, `weight_decay`.
ÚfusedT)r   z%optimizer got an empty parameter listr   Nz.CPU Offload currently only supports CUDA & XPUc                 óÄ  <€ V P                   Ee=   SP                  V ,          pSP                  P                  \	        \
        SP                  4      P                  4       4       \	        \
        SP                  4      P                  SP                  4      ;_uu_ 4        VP                   P                  V P                   RR7       R R R 4       V SP                  9   d   SP                  V  SP                  P                  4       SP                  V &   S'       d/   V P                   P                  SP                  4       R V n         R # R # R #   + '       g   i     L�; i©NT©Únon_blocking)ÚgradÚparam_d2h_mapÚstreamÚwait_streamÚgetattrÚtorchÚdeviceÚcurrent_streamÚcopy_ÚqueueÚrecord_eventÚrecord_stream)Úp_deviceÚp_hostr	   Úselfs   & €€r   Úbackward_hookÚ3CPUOffloadOptimizer.__init__.<locals>.backward_hookA   sñ   ø€ Ø�}‰}Ó(Ø×+Ñ+¨HÕ5�ð —‘×'Ñ'¬´°t·{±{Ó(C×(RÑ(RÓ(TÔUÜœU D§K¡KÓ0×7Ñ7¸¿¹×DÕDØ—K‘K×%Ñ% h§m¡mÀ$Ð%ÔG÷ Eð ˜tŸz™zÔ)ØŸ
™
 8Ð,Ø'+§{¡{×'?Ñ'?Ó'A�—
‘
˜8Ñ$÷ %Ø—M‘M×/Ñ/°·±Ô<Ø$(�H–Mñ %ñ! )÷
 E×Dús   Â (EÅE	Úcpu)r$   Ú
pin_memory)r0   r   éÿÿÿÿ)ÚcudaÚxpu)r#   ÚoptimÚAdamWÚupdateÚlistÚlenÚ
ValueErrorÚ
isinstanceÚdictr
   Úd_optÚd_param_groupsr   Ú
optim_dictr   r$   r"   ÚStreamr    r'   ÚpopÚrequires_gradÚnumelÚappendÚ
empty_liker   r&   ÚdetachÚ"register_post_accumulate_grad_hook)r,   r   r   r	   r
   ÚkwargsÚparam_groupsr-   Úparam_groupÚretained_paramsr*   r+   s   f&&d$,      r   Ú__init__ÚCPUOffloadOptimizer.__init__   s?  ù€ ð( œeŸk™k×/Ñ/Ó/°GÀ6Ô4IØ�M‰M ˆMÔ%ä˜F“|ˆÜˆ|Ó Ô!ÜÐDÓEÐEÜ˜, q�/¬4×0Ò0Ø% |Ð4Ð5ˆLð )ˆÔØˆŒ
Ø ˆÔä!›VˆÔÜ›&ˆŒÜ+Ó-¨bÕ1ˆŒØ�{‰{ð 
ô 
ð 	<ð <ó	<ð 
ô œe T§[¡[Ó1×8Ñ8Ó:ˆŒô “VˆŒ
ö	)ô* (ˆKØ —_‘_ XÓ.ˆFØ ˆOã"�Ø×-×-Ð-Ùà—>‘>Ó# d×&7Ñ&7Ô7Ø#×*Ñ*¨8Ô4Ùô ×)Ò)¨(¸5ÈTÔR�Ü#×.Ò.¨vÀ$ÔG�”à—‘˜XŸ_™_Ó.¸T�ÔBØ/5�×"Ñ" 8Ñ,à×;Ñ;¸MÔJÙ,;Ø Ð6¨+Ð6Ð7ñ-Ø;Añ-�—‘ Ó)ñ! #ô( �?Ó# a×'Ø×#Ñ#×*Ñ*¨H°oÐ+UÈÐ+U×Vñ3 (ô6 ˆt×"Ñ"Ó# aÔ'Ù(¨×)<Ñ)<ÑGÀÑGˆDŽJñ (r   Nc                ó|  € R pVe   V! 4       pV P                   e   V P                   P                  4        V P                  P                  4        F   w  r4VP	                  4        V P
                  V,          P                  4        V P                  V,          p\        \        V P                  4      P                  V P                  4      ;_uu_ 4        VP                  VRR7       R R R 4       K¢  	  V P                  P	                  4        V P                  P                  4        V#   + '       g   i     Kë  ; ir   )r<   Ústepr'   ÚitemsÚsynchronizer>   r   r"   r#   r$   r    r&   Úclear)r,   ÚclosureÚlossr*   Úgrad_d2h_eventr+   s   &&    r   rN   ÚCPUOffloadOptimizer.stept   sè   € àˆØÒÙ“9ˆDð �:‰:Ò!Ø�J‰J�O‰OÔà(,¯
©
×(8Ñ(8Ö(:Ñ$ˆHØ×&Ñ&Ô(Ø�O‰O˜HÕ%×*Ñ*Ô,ð
 ×'Ñ'¨Õ1ˆFÜœ §¡Ó,×3Ñ3°D·K±K×@Õ@Ø—‘˜v°D�Ô9÷ AÑ@ñ );ð 	�‰×ÑÔ!Ø�
‰
×ÑÔØˆ÷ A×@Ð@ús   ÃD*Ä*D;c                óÂ   € V'       g   Q hV P                   P                  4        F
  pR Vn        K  	  V P                  e   V P                  P	                  VR7       R # R # )N)Úset_to_none)r   Úkeysr   r<   Ú	zero_grad)r,   rW   r*   s   && r   rY   ÚCPUOffloadOptimizer.zero_gradŽ   sR   € ßÐˆ{ð ×*Ñ*×/Ñ/Ö1ˆHØ ˆHŽMñ 2ð �:‰:Ò!Ø�J‰J× Ñ ¨[Ð Ö9ñ "r   c                ón   € \        R  V P                  P                  4        4       V P                  R7      # )c              3   ó8   "  € T F  qP                   x € K  	  R # 5i©N)rH   )Ú.0r4   s   & r   Ú	<genexpr>Ú3CPUOffloadOptimizer.param_groups.<locals>.<genexpr>�   s   é € ÐFÑ-E E×ÖÓ-Eùs   ‚)Ústart)Úsumr>   Úvaluesr=   )r,   s   &r   rH   Ú CPUOffloadOptimizer.param_groups˜   s/   € ô ÙF¨T¯_©_×-CÑ-CÔ-EÓFØ×%Ñ%ô
ð 	
r   c                óà   € R V P                   P                  4        Uu. uF  qP                  4       NK  	  up/pV P                  '       d   V P                  P                  4       VR&   V# u upi )Ú	offloadedú	on-device)r>   rc   Ú
state_dictr<   )r,   r4   rh   s   &  r   rh   ÚCPUOffloadOptimizer.state_dict¡   sa   € à¸$¿/¹/×:PÑ:PÔ:RÓSÑ:R°×*Ñ*Ö,Ñ:RÑSð
ˆ
ð �:�:ˆ:Ø&*§j¡j×&;Ñ&;Ó&=ˆJ�{Ñ#ØÐùò	 Ts   žA+c                ó   € \        V P                  P                  4       VR ,          4       F  w  r#VP                  V4       K  	  V P                  '       d%   V P                  P                  VR,          4       R# RV9   d   \        R4      hR# )rf   rg   zPloaded state dict has a 'on-device' parameter group not present in the optimizerN)Úzipr>   rc   Úload_state_dictr<   r9   )r,   rh   r4   Úoptim_state_dicts   &&  r   rl   Ú#CPUOffloadOptimizer.load_state_dict©   s{   € Ü'*Ø�O‰O×"Ñ"Ó$ j°Õ&=ö(
Ñ#ˆEð ×!Ñ!Ð"2Ö3ñ(
ð
 �:�:ˆ:Ø�J‰J×&Ñ& z°+Õ'>Ö?Ø˜JÔ&ÜØbóð ñ 'r   )r<   r=   r$   r
   r>   r   r'   r    r]   )T)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r#   r4   r5   rK   Úno_gradrN   rY   ÚpropertyrH   rh   rl   Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s}   ø‡ € ð ,1¯;©;×+<Ñ+<ñ_Hð
 #(ð_Hð !÷_Hó _HðB ‡]‚]ƒ_óó ðô2:ð ñ
ó ð
ò÷ð r   r   )	Útypingr   r#   Útorch.optim.optimizerr   r   Útorchao.utilsr   r   © r   r   Ú<module>r{      s!   ðõ ã ß 4å /ôb˜)ö br   