+
    ÆÜ³jßm  ã                   ó   € ^ RI HtHt ^ RIHt ^ RIHt ^ RIt^ RI	H
t
 ^ RIt^ RIt^ RIHt ^ RIHt ]P$                  ! ]4      t ! R R4      t ! R	 R
4      t ! R R]P.                  P0                  4      t ! R R]4      t ! R R]4      tR# )é    )ÚabcÚdefaultdict)Údeepcopy)ÚchainN)ÚOptional)Úsync_gpuc                   ó&   a € ] tR t^t o R tRtV tR# )ÚMockArgsc                ó<   € V F  p\        WW,          4       K  	  R # ©N)Úsetattr)ÚselfÚinitial_dataÚkeys   && Ú]/home/luca/screen-checking/.venv/lib/python3.14/site-packages/bitsandbytes/optim/optimizer.pyÚ__init__ÚMockArgs.__init__   s   € ÛˆCÜ�D˜|Õ0Ö1ó  ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r
   r
      s   ø‡ € ÷2ð 2r   r
   c                   óZ   a € ] tR t^t o RtRtR tR t]R 4       t	R t
R
R ltR tR	tV tR# )ÚGlobalOptimManagerzC
A global optimizer manager for enabling custom optimizer configs.
Nc                ó   € \        R 4      h)zCall get_instance() instead)ÚRuntimeError©r   s   &r   r   ÚGlobalOptimManager.__init__!   s   € ÜÐ8Ó9Ð9r   c                óL   € / V n         / V n        R V n        RV n        . V n        R # )NF)Ú
pid2configÚindex2configÚ	optimizerÚuses_config_overrideÚmodule_weight_config_tripler"   s   &r   Ú
initializeÚGlobalOptimManager.initialize$   s(   € ØˆŒØˆÔØˆŒØ$)ˆÔ!Ø+-ˆÖ(r   c                ó–   € V P                   f1   V P                  V 4      V n         V P                   P                  4        V P                   # r   )Ú	_instanceÚ__new__r*   )Úclss   &r   Úget_instanceÚGlobalOptimManager.get_instance+   s6   € à�=‰=Ò ØŸK™K¨Ó,ˆCŒMØ�M‰M×$Ñ$Ô&Ø�}‰}Ðr   c                óJ  € \        V4      p\        V^ ,          \        4      '       g   RV/.p\        V4       Ff  w  r4\        VR,          4       FK  w  rV\	        V4      V P
                  9   g   K!  V P
                  \	        V4      ,          V P                  W53&   KM  	  Kh  	  R# )r   ÚparamsN)ÚlistÚ
isinstanceÚdictÚ	enumerateÚidr%   r&   )r   r3   Úparam_groupsÚgroup_indexÚgroupÚp_indexÚps   &&     r   Úregister_parametersÚ&GlobalOptimManager.register_parameters2   sƒ   € Ü˜F“|ˆÜ˜, q�/¬4×0Ò0Ø% |Ð4Ð5ˆLä"+¨LÖ"9ÑˆKÜ'¨¨h­Ö8‘
�Ü�a“5˜DŸO™OÖ+Ø@DÇÁÔPRÐSTÓPUÕ@V�D×%Ñ% {Ð&<Ó=ó 9ó #:r   c                ó°  € RV n         \        V\        P                  P                  4      '       d   V.p\        V\        P
                  4      '       d   V.pVe   Ve
   Ve   Q hW#/pVej   V Fa  p\        V4      V P                  9   d.   V P                  \        V4      ,          P                  V4       KJ  W@P                  \        V4      &   Kc  	  R# R# )aú  
Override initial optimizer config with specific hyperparameters.

The key-values of the optimizer config for the input parameters are overridden
This can be both, optimizer parameters like `betas` or `lr`, or it can be
8-bit specific parameters like `optim_bits`.

Arguments:
   parameters (`torch.Tensor` or `list(torch.Tensors)`):
     The input parameters.
   key (`str`):
     The hyperparameter to override.
   value:
     The hyperparameter value.
   key_value_dict (`dict`):
     A dictionary with multiple key-values to override.

Example:

```py
import torch
import bitsandbytes as bnb

mng = bnb.optim.GlobalOptimManager.get_instance()

model = MyModel()
mng.register_parameters(model.parameters()) # 1. register parameters while still on CPU

model = model.cuda()
# use 8-bit optimizer states for all parameters
adam = bnb.optim.Adam(model.parameters(), lr=0.001, optim_bits=8)

# 2. override: the parameter model.fc1.weight now uses 32-bit Adam
mng.override_config(model.fc1.weight, 'optim_bits', 32)
```
TN)	r(   r5   ÚtorchÚnnÚ	ParameterÚTensorr8   r%   Úupdate)r   Ú
parametersr   ÚvalueÚkey_value_dictr=   s   &&&&& r   Úoverride_configÚ"GlobalOptimManager.override_config<   s°   € ðJ %)ˆÔ!Ü�j¤%§(¡(×"4Ñ"4×5Ò5Ø$˜ˆJÜ�j¤%§,¡,×/Ò/Ø$˜ˆJØŠ?˜uÒ0Ø!Ò)Ð)Ð)Ø!˜\ˆNàÒ%Û�Ü�a“5˜DŸO™OÔ+Ø—O‘O¤B q£EÕ*×1Ñ1°.ÖAà-;—O‘O¤B q£EÓ*ó	  ñ &r   c                ó@   € V P                   P                  WV34       R # r   )r)   Úappend)r   ÚmoduleÚ
param_nameÚconfigs   &&&&r   Úregister_module_overrideÚ+GlobalOptimManager.register_module_overrideq   s   € Ø×(Ñ(×/Ñ/°ÀVÐ0LÖMr   )r&   r)   r'   r%   r(   )NNN)r   r   r   r   Ú__doc__r-   r   r*   Úclassmethodr0   r>   rI   rP   r   r   r   s   @r   r   r      sG   ø‡ € ñð €Iò:ò.ð ñó ðòWô3<÷jNð Nr   r   c                   óØ   a a€ ] tR t^ut oRtRV 3R lltR tV 3R ltV 3R ltRR lt	R t
R t]P                  ! 4       RR	 l4       tR
 tR tR t]P$                  3R ltR tRtVtV ;t# )ÚOptimizer8bitÚ__bnb_optimizer_quant_state__c                ó
  <€ \         SV `  W4       RV n        / V n        W@n        \
        P                  P                  4       V n        \        P                  4       V n
        0 RmV n        V^8X  d   V P                  4        R# R# )a2  
Base 8-bit optimizer class.

Arguments:
    params (`torch.Tensor`):
        The input parameters to optimize.
    optim_bits (`int`, defaults to 32):
        The number of bits of the optimizer state.
    is_paged (`bool`, defaults to `False`):
        Whether the optimizer is a paged optimizer or not.
FN>   Úmax1Úmax2Úqmap1Úqmap2Ústate1Ústate2Úabsmax1Úabsmax2Únew_max1Únew_max2Ú	gnorm_vecÚ	unorm_vec)Úsuperr   ÚinitializedÚ	name2qmapÚis_pagedÚFÚGlobalPageManagerr0   Úpage_mngr   ÚmngÚnon_castable_tensor_keysÚ	fill_qmap)r   r3   ÚdefaultsÚ
optim_bitsrg   Ú	__class__s   &&&&&€r   r   ÚOptimizer8bit.__init__x   sn   ø€ ô 	‰Ñ˜Ô*Ø ˆÔØˆŒØ ŒÜ×+Ñ+×8Ñ8Ó:ˆŒä%×2Ñ2Ó4ˆŒò)
ˆÔ%ð ˜Œ?Ø�N‰NÖñ r   c                ó–   € \         P                  ! R R7      V P                  R&   \         P                  ! RR7      V P                  R&   R# )T)ÚsignedÚdynamicFÚudynamicN)rh   Úcreate_dynamic_maprf   r"   s   &r   rm   ÚOptimizer8bit.fill_qmap�   s4   € Ü$%×$8Ò$8ÀÔ$Eˆ�‰�yÑ!Ü%&×%9Ò%9ÀÔ%Gˆ�‰�zÓ"r   c                óB  <€ \         S
V `  4       pVR,          P                  4        UUUUu/ uFB  w  r#T\        V\        4      '       d%   VP                  4        UUu/ uF  w  rEWEbK	  	  uppMTbKD  	  uppppVR&   VR,          P                  4        Ft  p\        V\        4      '       g   K  / pV Uu. uF  q"V P                  9   g   K  VNK  	  ppV F  p	VP                  V	4      Wy&   K  	  V'       g   Kf  WvV P                  &   Kv  	  V# u uppi u uppppi u upi )ae  Return optimizer state, wrapping quantization tensors for FSDP compatibility.

FSDP's full_optim_state_dict gathers all tensor states across ranks.
Quantization states (state1, state2, absmax, etc.) have different shapes
than model parameters, causing gather operations to fail. By wrapping
these tensors in a nested dict, FSDP skips them during gathering.
Ústate)	rd   Ú
state_dictÚitemsr5   r6   Úvaluesrl   ÚpopÚ_FSDP_WRAPPED_QUANT_STATE_KEY)r   rz   ÚkÚvÚkkÚvvÚparam_stateÚquant_stateÚkeys_to_wrapr   rp   s   &         €r   rz   ÚOptimizer8bit.state_dict¡   s  ø€ ô ‘WÑ'Ó)ˆ
ð
 [eÐelÕZm×ZsÑZsÔZuö
ÙZuÑRVÐRSˆA´:¸aÄ×3FÒ3F Q§W¡W¤YÔ/¡Y™6˜2�’¡YÓ/ÈAÒMÑZuô
ˆ
�7Ñð
 & gÕ.×5Ñ5Ö7ˆKÜ˜+¤t×,Ô,Ø �Ù+6Ó]©; a¸t×?\Ñ?\Ñ:\§ ©;�Ð]Û'�CØ'2§¡°sÓ';�KÓ$ñ (ç‘;ØFQ × BÑ BÓCñ 8ð Ðùó 0ùõ
ùò  ^s#   ¬1D
ÁDÁ*
D
Â5DÃDÄD
c                ó&   <€ \         SV `  V4       R # r   )rd   Ú__setstate__)r   ry   rp   s   &&€r   rˆ   ÚOptimizer8bit.__setstate__½   s   ø€ Ü‰Ñ˜UÖ#r   c                ó8  a aa€ \        V4      pVR,          P                  4        FZ  p\        V\        4      '       g   K  S P                  V9   g   K.  VP                  S P                  4      pVP                  V4       K\  	  S P                  pVR,          p\        V4      \        V4      8w  d   \        R4      hR V 4       pR V 4       p\        ;QJ d%    R \        Wx4       4       F  '       g   K   RM	  RM! R \        Wx4       4       4      '       d   \        R	4      h\        \        P                  ! R
 V 4       4      \        P                  ! R V 4       4      4       U	U
u/ uF  w  ršWšbK	  	  pp	p
VVV 3R lo\        \        4      pVR,          P                  4        F#  w  rÞWÛ9   d   W½,          pS! Wþ4      WÏ&   K  WìV&   K%  	  R p\        WV4       UUu. uF  w  ppV! VV4      NK  	  pppS P!                  RVRV/4       R# u up
p	i u uppi )a  Load an optimizer state.

Arguments:
    state_dict (`dict`):
        An optimizer state (should be returned from a call to `state_dict`) to load.
    move_to_device (`bool`, defaults to `True`):
        Whether to move the optimizer's state to the device.
ry   r9   z<loaded state dict has a different number of parameter groupsc              3   óF   "  € T F  p\        VR ,          4      x € K  	  R# 5i©r3   N©Úlen©Ú.0Úgs   & r   Ú	<genexpr>Ú0Optimizer8bit.load_state_dict.<locals>.<genexpr>Ø   s   é € Ð7±¨1”c˜!˜H�+×&Ð&³ùó   ‚!c              3   óF   "  € T F  p\        VR ,          4      x € K  	  R# 5irŒ   r�   r�   s   & r   r’   r“   Ù   s   é € Ð=±¨1”c˜!˜H�+×&Ð&³ùr”   c              3   ó.   "  € T F  w  rW8g  x € K  	  R # 5ir   r   )r�   Úp_lenÚs_lens   &  r   r’   r“   Ú   s   é € ÐNÑ2M¡, %ˆuŽ~Ó2Mùs   ‚TFz]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc              3   ó2   "  € T F  qR ,          x € K  	  R# 5irŒ   r   r�   s   & r   r’   r“   ã   s   é € Ð#F¹°A h§K¢K»ùó   ‚c              3   ó2   "  € T F  qR ,          x € K  	  R# 5irŒ   r   r�   s   & r   r’   r“   ä   s   é € Ð#@¹°A h§K¢K»ùrš   c                óR  <a € \        V\        P                  4      '       dS   S P                  4       '       d;   VP                  \        P
                  8w  d   VP                  S P                  4      pV# \        V\        4      '       da   VP                  4        FJ  w  r#VSP                  9   d*   S'       d    VP                  S P                  4      W&   K=  K?  S! S V4      W&   KL  	  V# \        V\        P                  4      '       d   \        V4      ! VV 3R lV 4       4      # V# )zBMake a deep copy of value, casting all tensors to device of param.c              3   ó6   <"  € T F  pS! SV4      x € K  	  R # 5ir   r   )r�   r€   ÚcastÚparams   & €€r   r’   Ú>Optimizer8bit.load_state_dict.<locals>.cast.<locals>.<genexpr>ú   s   øé € Ð"A¹5°a¡4¨¨q§> >»5ùs   ƒ)r5   rA   rD   Úis_floating_pointÚdtypeÚuint8Útor6   r{   rl   ÚdeviceÚcontainer_abcsÚIterableÚtype)rŸ   rG   r   r€   rž   Úmove_to_devicer   s   f&  €€€r   rž   Ú+Optimizer8bit.load_state_dict.<locals>.castè   s×   ù€ ä˜%¤§¡×.Ò.ð ×*Ñ*×,Ò,°·±ÄÇÁÔ1KØ!ŸH™H U§[¡[Ó1�EØ�Ü˜E¤4×(Ò(Ø!ŸK™KžM‘D�AØ˜D×9Ñ9Ô9ß)Ø'(§t¡t¨E¯L©LÓ'9˜E›Hñ *ñ $(¨¨q£>˜›ñ *ð �Ü˜E¤>×#:Ñ#:×;Ò;Ü˜E”{Õ"A¹5Ó"AÓAÐAà�r   c                 ó   € V R ,          VR &   V# )r3   r   )r;   Ú	new_groups   &&r   Úupdate_groupÚ3Optimizer8bit.load_state_dict.<locals>.update_group
  s   € Ø"'¨¥/ˆI�hÑØÐr   N)r   r|   r5   r6   r~   r}   rE   r9   rŽ   Ú
ValueErrorÚanyÚzipr   Úfrom_iterabler   r{   rˆ   )r   rz   r©   rƒ   r„   ÚgroupsÚsaved_groupsÚ
param_lensÚ
saved_lensÚold_idr=   Úid_mapry   r   r€   rŸ   r­   r‘   Úngr9   rž   s   f&f                 @r   Úload_state_dictÚOptimizer8bit.load_state_dictÀ   sÚ  ú€ ô ˜jÓ)ˆ
ð & gÕ.×5Ñ5Ö7ˆKÜ˜+¤t×,Ô,°×1SÑ1SÐWbÖ1bØ)Ÿo™o¨d×.PÑ.PÓQ�Ø×"Ñ" ;Ö/ñ 8ð ×"Ñ"ˆØ! .Õ1ˆäˆv‹;œ#˜lÓ+Ô+ÜÐ[Ó\Ð\Ù7±Ó7ˆ
Ù=±Ó=ˆ
ß‹3ÑN´#°jÔ2MÓN�3�3Š3ÑN´#°jÔ2MÓN×NÒNÜØoóð ô !Ü×#Ò#Ñ#F¹Ó#FÓFÜ×#Ò#Ñ#@¹Ó#@Ó@ôô
ñ‘	�ð ŠIñð 	ñ 
÷	ô2 œDÓ!ˆØ˜wÕ'×-Ñ-Ö/‰DˆAØŒ{Ø�	�Ù# E›~�“à�a“ñ 0ò	ô :=¸VÔ9RÔSÑ9R±°°2™ Q¨Ö+Ñ9RˆÑSØ×Ñ˜7 E¨>¸<ÐHÖIùó_
ùó\ Ts   Å+HÇ!Hc                óþ  € \        V P                  4       Fã  w  r\        VR ,          4       FÈ  w  r4VP                  P                  R8X  d   K"  W@P                  9   g   K4  V P                  V,          pVP                  4        Fm  w  rg\        V\        P                  4      '       g   K'  \        VRR4      pV'       d   K>  VP                  VP                  4      V P                  V,          V&   Ko  	  KÊ  	  Kå  	  R# )r3   Úcpurg   FN)r7   r9   r¥   r¨   ry   r{   r5   rA   rD   Úgetattrr¤   )	r   Úgindexr;   Úpindexr=   r|   r   r€   rg   s	   &        r   Úto_gpuÚOptimizer8bit.to_gpu  s«   € Ü& t×'8Ñ'8Ö9‰MˆFÜ& u¨X¥Ö7‘	�Ø—8‘8—=‘= EÔ)ÙØŸ
™
–?Ø!ŸZ™Z¨�]�FØ &§¡¦™˜Ü% a¬¯©×6Ô6Ü'.¨q°*¸eÓ'D˜Hß#+¡8Ø34·4±4¸¿¹³> §
¡
¨1¥¨aÓ 0ó	 !/ó 8ó :r   c                óŠ  € V P                   P                   EF'  w  rp\        W4      pVf   Q h\        V\        P
                  4      '       g#   \        V\        P                  4      '       g   Q hRp\        V P                  4       F²  w  rgV'       d    K�  \        VR,          4       FŒ  w  r‰V'       d    K2  \        V	4      \        V4      8X  g   K+  W0P                   P                  \        V	4      &   V P                   P                  \        V	4      ,          V P                   P                  Wh3&   RpKŽ  	  K´  	  EK*  	  R # )NFr3   T)rk   r)   r¾   r5   rA   rD   rC   r7   r9   r8   r%   r&   )
r   rM   ÚattrrO   ÚpmoduleÚfoundr¿   r;   rÀ   r=   s
   &         r   Úcheck_overridesÚOptimizer8bit.check_overrides  sð   € Ø$(§H¡H×$HÕ$HÑ ˆF˜&Ü˜fÓ+ˆGØÒ&Ð&Ð&Ü˜g¤u§|¡|×4Ò4¼
À7ÌEÏOÉO×8\Ò8\Ð\Ð\ØˆEÜ!*¨4×+<Ñ+<Ö!=‘�ßÚÜ!*¨5°­?Ö!;‘I�FßÚÜ˜!“u¤ 7£Ö+ð 6<Ÿ™×+Ñ+¬B¨q«EÑ2ØBFÇ(Á(×BUÑBUÔVXÐYZÓV[ÕB\˜Ÿ™×-Ñ-¨vÐ.>Ñ?Ø $šó "<ô ">ó %Ir   c                ó¤  € RpVe.   \         P                  ! 4       ;_uu_ 4        V! 4       pRRR4       V P                  '       g(   V P                  4        V P	                  4        RV n        Rp\        V P                  4       F“  w  rE\        VR,          4       Fx  w  rcVP                  f   K  V P                  V,          p\        V4      ^ 8X  d   V P                  WSWF4       V P                  V4       V P                  WSWF4       \        V4       Kz  	  K•  	  V P                  '       d   Ve   \        V4       V#   + '       g   i     EL; i)z­Perform a single optimization step.

Arguments:
    closure (`Callable`, *optional*, defaults to `None`):
        A closure that reevaluates the model and returns the loss.
NTr3   )rA   Úenable_gradre   rÇ   rÁ   r7   r9   Úgradry   rŽ   Ú
init_stateÚprefetch_stateÚupdate_stepr   rg   )r   ÚclosureÚlossr=   r¿   r;   rÀ   ry   s   &&      r   ÚstepÚOptimizer8bit.step1  s  € ð ˆØÒÜ×"Ò"×$Õ$Ù“y�÷ %ð ××ÐØ× Ñ Ô"Ø�K‰KŒMØ#ˆDÔð ˆÜ& t×'8Ñ'8Ö9‰MˆFÜ& u¨X¥Ö7‘	�Ø—6‘6’>ÙØŸ
™
 1��Ü�u“: ”?Ø—O‘O E¨fÔ=à×#Ñ# AÔ&Ø× Ñ  ¨6Ô:Ü˜–ó 8ñ :ð �=�=ˆ=˜Qš]ô �QŒKàˆ÷5 %×$Ð$ús   ¤D>Ä>E	c                ó6  € / pVR ,          VR &   VR,          VR&   VR,          VR&   VR,          VR&   VP                  RR4      VR&   VP                  RR4      VR&   VP                  RR4      VR&   V P                  P                  VR	&   V P                  P                  VR
&   V P                  P                  VR&   V P                  P
                  VR&   W3V P                  P                  9   d.   VP                  V P                  P                  W3,          4       V P                  V,          R,          V,          p\        V4      V P                  P                  9   d6   VP                  V P                  P                  \        V4      ,          4       V# )ÚbetasÚepsÚweight_decayÚlrÚalphaç        Út_alphaNÚt_beta3ro   Úmin_8bit_sizeÚ	max_unormÚ
skip_zerosr3   )ÚgetÚargsro   rÜ   rÝ   rÞ   rk   r&   rE   r9   r8   r%   )r   r¿   rÀ   r;   rO   r=   s   &&&&  r   Ú
get_configÚOptimizer8bit.get_configW  sT  € ØˆØ �.ˆˆw‰Ø˜e�ˆˆu‰Ø!& ~Õ!6ˆˆ~ÑØ˜T•{ˆˆt‰ØŸ)™) G¨SÓ1ˆˆw‰Ø!ŸI™I i°Ó6ˆˆyÑØ!ŸI™I i°Ó6ˆˆyÑØ#Ÿy™y×3Ñ3ˆˆ|ÑØ"&§)¡)×"9Ñ"9ˆˆÑØ"Ÿi™i×1Ñ1ˆˆ{ÑØ#Ÿy™y×3Ñ3ˆˆ|ÑàÐ˜tŸx™x×4Ñ4Ô4Ø�M‰M˜$Ÿ(™(×/Ñ/°Ð0@ÕAÔBð ×Ñ˜fÕ% hÕ/°Õ7ˆÜˆa‹5�D—H‘H×'Ñ'Ô'Ø�M‰M˜$Ÿ(™(×-Ñ-¬b°«eÕ4Ô5àˆr   c                ó   € \        R 4      h)z(init_state method needs to be overridden©ÚNotImplementedError©r   r;   r=   r¿   rÀ   s   &&&&&r   rÌ   ÚOptimizer8bit.init_statep  s   € Ü!Ð"LÓMÐMr   c                ó   € \        R 4      h)z-The update_step method needs to be overriddenrä   ræ   s   &&&&&r   rÎ   ÚOptimizer8bit.update_steps  s   € Ü!Ð"QÓRÐRr   c                ój  € VP                   P                  R 8X  dg   V P                  '       d3   \        V RR4      '       g    \        P
                  ! R^R7       RV n        \        P                  ! WVP                   R7      # V P                  '       d   VP                  4       R8  d#   \        P                  ! WVP                   R7      # \        P                  ! VP                  RVR	VP                   / p\        P                  ! V^ 4       V P                  P                  P!                  V4       V# )
r½   Ú_cpu_paged_warnedFzXPaged optimizers are not supported on CPU. Falling back to non-paged optimizer behavior.)Ú
stacklevelT©r¢   r¥   g     jø@r¢   r¥   )r¥   r¨   rg   r¾   ÚwarningsÚwarnrë   rA   Ú
zeros_likeÚnumelrh   Ú	get_pagedÚshapeÚfillrj   Úpaged_tensorsrL   )r   r=   r¢   Úbuffs   &&& r   Úget_state_bufferÚOptimizer8bit.get_state_bufferv  sÕ   € Ø�8‰8�=‰=˜EÔ!Ø�}�}ˆ}¤W¨TÐ3FÈ×%NÒ%NÜ—’ØnØ õð *.�Ô&Ü×#Ò# A¸1¿8¹8ÔDÐDØ�}�}ˆ} §¡£	¨C¤Ü×#Ò# A¸1¿8¹8ÔDÐDô —;’; §¡ÐF¨uÐF¸Q¿X¹XÑFˆDÜ�FŠF�4˜ŒOØ�M‰M×'Ñ'×.Ñ.¨tÔ4ØˆKr   c                ó  € V P                   '       dy   V P                  V,          pVR ,          p\        VRR4      pV'       dF   \        P                  ! VR ,          4       RV9   d    \        P                  ! VR,          4       R# R# R# R# )r\   rg   Fr]   N)rg   ry   r¾   rh   Úprefetch_tensor)r   r=   ry   Ús1rg   s   &&   r   rÍ   ÚOptimizer8bit.prefetch_stateˆ  sm   € Ø�=�=ˆ=Ø—J‘J˜q•MˆEØ�x•ˆBÜ˜r :¨uÓ5ˆHßÜ×!Ò! %¨¥/Ô2Ø˜uÔ$Ü×%Ò% e¨H¥oÖ6ñ %ñ ñ	 r   )rë   re   rg   rk   rf   rl   rj   )é    F)Tr   )r   r   r   r   r~   r   rm   rz   rˆ   rº   rÁ   rÇ   rA   Úno_gradrÑ   rá   rÌ   rÎ   Úfloat32r÷   rÍ   r   r   Ú__classcell__©rp   r   s   @@r   rU   rU   u   sx   ù‡ € Ø$CÐ!÷#òJHõõ8$ôOJòbBò%ð& ‡]‚]ƒ_ó#ó ð#òJò2NòSð ).¯©ô ÷$7ò 7r   rU   c                   óž   a a€ ] tR tRt oRV3R lV 3R lllt]P                  ! 4       R 4       t]P                  ! 4       R 4       tRt	Vt
V ;t# )ÚOptimizer2Statei“  c                óF   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          /# )é   rÚ   rÛ   )r   Úint)Úformatr   s   "€r   Ú__annotate__ÚOptimizer2State.__annotate__”  s-   ø€ ÷ U-ñ U-ñ ™#•ðU-ñ  ™#•ñ!U-r   c           
     óª  <€ RV8:  g   \        RV 24      hRV8:  g   \        RV 24      h\        V\        4      '       dZ   VP                  RR4      P                  RR4      P	                  4       P                  R4      pV Uu. uF  p\        V4      NK  	  pp\        \        V4      4       F3  pRVV,          u;8:  d
   R8  d   K  M \        R	V R
VV,           24      h	  RV8:  g   \        RV 24      h\        W4WVWÞVR7      p\        SV `-  VVW|4       Vf$   / pWxR&   W˜R&   W¨R&   W¸R&   \        V4      V n        MW€n        Wn        R# u upi )aÐ  
Base 2-state update optimizer class.

Arguments:
    optimizer_name (`str`):
        The name of the optimizer.
    params (`torch.Tensor`):
        The input parameters to optimize.
    lr (`float`, defaults to 1e-3):
        The learning rate.
    betas (`tuple`, defaults to (0.9, 0.999)):
        The beta values for the optimizer.
    eps (`float`, defaults to 1e-8):
        The epsilon value for the optimizer.
    weight_decay (`float`, defaults to 0.0):
        The weight decay value for the optimizer.
    optim_bits (`int`, defaults to 32):
        The number of bits of the optimizer state.
    args (`object`, defaults to `None`):
        An object with additional arguments.
    min_8bit_size (`int`, defaults to 4096):
        The minimum number of elements of the parameter tensors for 8-bit optimization.
    max_unorm (`float`, defaults to 0.0):
        The maximum value to normalize each block with.
    skip_zeros (`bool`, defaults to `False`):
        Whether to skip zero values for sparse gradients and models to ensure correct updates.
    is_paged (`bool`, defaults to `False`):
        Whether the optimizer is a paged optimizer or not.
    alpha (`float`, defaults to 0.0):
        The alpha value for the AdEMAMix optimizer.
    t_alpha (`Optional[int]`, defaults to `None`):
        Number of iterations for alpha scheduling with AdEMAMix.
    t_beta3 (`Optional[int]`, defaults to `None`):
        Number of iterations for beta scheduling with AdEMAMix.

rÙ   úInvalid learning rate: úInvalid epsilon value: Ú(Ú Ú)Ú,ç      ð?ú Invalid beta parameter at index ú: úInvalid weight_decay value: )r×   rÔ   rÕ   rÖ   rØ   rÚ   rÛ   Nro   rÜ   rÝ   rÞ   )r¯   r5   ÚstrÚreplaceÚstripÚsplitÚfloatÚrangerŽ   r6   rd   r   r
   rà   Úoptimizer_name)r   r  r3   r×   rÔ   rÕ   rÖ   ro   rà   rÜ   rÝ   rÞ   rg   rØ   rÚ   rÛ   ÚbÚirn   rp   s   &&&&&&&&&&&&&&&&   €r   r   ÚOptimizer2State.__init__”  s]  ø€ ðl �bŒyÜÐ6°r°dÐ;Ó<Ð<Ø�cŒzÜÐ6°s°eÐ<Ó=Ð=Ü�eœS×!Ò!à—M‘M # rÓ*×2Ñ2°3¸Ó;×AÑAÓC×IÑIÈ#ÓNˆEÙ',Ó-¡u !”U˜1–X¡uˆEÐ-Ü”s˜5“zÖ"ˆAØ˜% �(Ö( S×(Ð(Ü Ð#CÀAÀ3ÀbÈÈqÍÈ
Ð!SÓTÐTñ #ð �lÔ"ÜÐ;¸L¸>ÐJÓKÐKäØ CÈ%Ðjqô
ˆô 	‰Ñ˜ ¨:Ô@àŠ<ØˆDØ!+�ÑØ$1�Ñ!Ø )�ÑØ!+�Ñä  ›ˆD�IàŒIà,Öùò1 .s   ÂEc                ó
  € V P                  W4V4      pVR ,          ^ 8X  d   \        P                  pM4VR ,          ^8X  d   \        P                  pM\	        RVR ,           24      hVP                  4       VR,          8  d   \        P                  pV P                  V,          p^ VR&   V\        P                  8X  dK   V P                  V\        P                  R7      VR&   V P                  V\        P                  R7      VR&   EMÅV\        P                  8X  Ed°   VR,          ^ 8X  d”   RV P                  9  d   V P                  4        V P                  R,          P                  VP                  4      V P                  R&   V P                  R,          P                  VP                  4      V P                  R&   V P                  V\        P                  R7      VR&   V P                  R,          VR	&   V P                  V\        P                  R7      VR&   V P                  R,          VR
&   RpVP                  4       p	W˜,          \        W˜,          4      ,           p
\        P                  ! V
3\        P                  VP                  R7      VR&   \        P                  ! V
3\        P                  VP                  R7      VR&   VR,          R8”  d(   \        P                  ! RVP                  R7      VR&   R# R# )ro   ú(Amount of optimizer bits not supported: rÜ   rÑ   ©r¢   r\   r]   rt   ru   rZ   r[   é   rí   r^   r_   rÝ   rÙ   ©r¥   rc   N©é   ©rá   rA   rÿ   r£   rå   rñ   ry   r÷   rf   rm   r¤   r¥   ÚboolÚzeros©r   r;   r=   r¿   rÀ   rO   r¢   ry   Ú	blocksizeÚnÚblockss   &&&&&      r   rÌ   ÚOptimizer2State.init_stateë  s?  € à—‘ °Ó7ˆà�,Õ 2Ô%Ü—M‘M‰EØ�LÕ! QÔ&Ü—K‘K‰Eä%Ð(PÐQWÐXdÕQeÐPfÐ&gÓhÐhà�7‰7‹9�v˜oÕ.Ô.Ü—M‘MˆEà—
‘
˜1•ˆØˆˆf‰à”E—M‘MÔ!Ø"×3Ñ3°A¼U¿]¹]Ð3ÓKˆE�(‰OØ"×3Ñ3°A¼U¿]¹]Ð3ÓKˆE�(‹OØ”e—k‘kÕ!Ø�V�} Ô!Ø D§N¡NÔ2Ø—N‘NÔ$Ø,0¯N©N¸9Õ,E×,HÑ,HÈÏÉÓ,R�—‘˜yÑ)Ø-1¯^©^¸JÕ-G×-JÑ-JÈ1Ï8É8Ó-T�—‘˜zÑ*à"×3Ñ3°A¼U¿[¹[Ð3ÓIˆE�(‰OØ!Ÿ^™^¨IÕ6ˆE�'‰Nà"×3Ñ3°A¼U¿[¹[Ð3ÓIˆE�(‰OØ!Ÿ^™^¨JÕ7ˆE�'‰NàˆIØ—‘“	ˆAØ•n¬¨Q­]Ó(;Õ;ˆFä$Ÿ{š{¨F¨9¼E¿M¹MÐRS×RZÑRZÔ[ˆE�)ÑÜ$Ÿ{š{¨F¨9¼E¿M¹MÐRS×RZÑRZÔ[ˆE�)Ñà�+Õ Ô$Ü!&§¢¨T¸!¿(¹(Ô!CˆE�+Óñ %r   c                óð  € VP                   P                  4       Vn         VP                  P                  4       Vn        V P                  V,          pVP                  pV P	                  W4V4      pVR ;;,          ^,          uu&   VR ,          pVR,          P
                  \        P                  8X  dÏ   \        P                  ! V P                  TTVR,          VR,          ^ ,          VR,          TVR,          VR,          VR,          ^,          \        VR,          4      ^8¼  d   VR,          ^,          MRVP                  RR4      VR,          R	VR
,          R8”  d
   VR,          MRVR
,          VR,          R7       R# VR,          P
                  \        P                  8X  dÏ   \        P                  ! V P                  TTVR,          VR,          VR,          ^ ,          VR,          ^,          \        VR,          4      ^8¼  d   VR,          ^,          MRVP                  RR4      VR,          VVR,          VR,          VR,          VR,          VR,          VR,          R	VR,          R7       R# R# )rÑ   r\   rÔ   rÕ   r×   r]   rÙ   rØ   rÖ   r  rÝ   rc   NrÞ   ©rÝ   rÞ   rZ   r[   r^   r_   ©Úgnorm_scalerÞ   )ÚdataÚ
contiguousrË   ry   rá   r¢   rA   r  rh   Úoptimizer_update_32bitr  rŽ   rß   r£   Úoptimizer_update_8bit_blockwise©	r   r;   r=   r¿   rÀ   ry   rË   rO   rÑ   s	   &&&&&    r   rÎ   ÚOptimizer2State.update_step  sþ  € ð —‘×"Ñ"Ó$ˆŒØ—‘×"Ñ"Ó$ˆŒà—
‘
˜1•ˆØ�v‰vˆà—‘ °Ó7ˆàˆf�˜Õ‹Ø�V�}ˆà��?× Ñ ¤E§K¡KÔ/Ü×$Ò$Ø×#Ñ#ØØØ�h•Ø�w• Õ"Ø�u•ØØ�t•Ø�h•Ø�w• Õ"Ü&)¨&°­/Ó&:¸aÔ&?��w• Ö"ÀSØ—
‘
˜7 CÓ(Ø�~Õ&ØØ&,¨[Õ&9¸CÔ&?��kÖ"ÀTØ  Õ-Ø! ,Õ/÷#ð( �8�_×"Ñ"¤e§k¡kÔ1Ü×-Ò-Ø×#Ñ#ØØØ�h•Ø�h•Ø�w• Õ"Ø�w• Õ"Ü&)¨&°­/Ó&:¸aÔ&?��w• Ö"ÀSØ—
‘
˜7 CÓ(Ø�u•ØØ�t•Ø�g•Ø�g•Ø�iÕ Ø�iÕ Ø�~Õ&ØØ! ,Õ/÷'ñ 2r   ©rà   r  )çü©ñÒMbP?)çÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>rÙ   rý   Né   rÙ   FFrÙ   NN©r   r   r   r   r   rA   rþ   rÌ   rÎ   r   r   r   r  s   @@r   r  r  “  sN   ù‡ € ÷U-õ U-ðn ‡]‚]ƒ_ñ(Dó ð(DðT ‡]‚]ƒ_ñ7ó ÷7ð 7r   r  c                   ó’   a a€ ] tR tRt oRV 3R llt]P                  ! 4       R 4       t]P                  ! 4       R 4       tRt	Vt
V ;t# )ÚOptimizer1StateiQ  c                ó¸  <€ RV8:  g   \        RV 24      hRV8:  g   \        RV 24      h\        \        V4      4       F1  pRWM,          u;8:  d
   R8  d   K  M \        RV RWM,           24      h	  RV8:  g   \        RV 24      h\        W4WVR7      p\        SV `  W.W|4       Vf$   / pWxR
&   W˜R&   W¨R&   W¸R&   \        V4      V n        MW€n        Wn        R	# )aŒ  
Base 1-state update optimizer class.

Arguments:
    optimizer_name (`str`):
        The name of the optimizer.
    params (`torch.Tensor`):
        The input parameters to optimize.
    lr (`float`, defaults to 1e-3):
        The learning rate.
    betas (`tuple`, defaults to (0.9, 0.0)):
        The beta values for the optimizer.
    eps (`float`, defaults to 1e-8):
        The epsilon value for the optimizer.
    weight_decay (`float`, defaults to 0.0):
        The weight decay value for the optimizer.
    optim_bits (`int`, defaults to 32):
        The number of bits of the optimizer state.
    args (`object`, defaults to `None`):
        An object with additional arguments.
    min_8bit_size (`int`, defaults to 4096):
        The minimum number of elements of the parameter tensors for 8-bit optimization.
    max_unorm (`float`, defaults to 0.0):
        The maximum value to normalize each block with.
    skip_zeros (`bool`, defaults to `False`):
        Whether to skip zero values for sparse gradients and models to ensure correct updates.
    is_paged (`bool`, defaults to `False`):
        Whether the optimizer is a paged optimizer or not.
rÙ   r  r  r  r  r  r  )r×   rÔ   rÕ   rÖ   Nro   rÜ   rÝ   rÞ   )	r¯   r  rŽ   r6   rd   r   r
   rà   r  )r   r  r3   r×   rÔ   rÕ   rÖ   ro   rà   rÜ   rÝ   rÞ   rg   r  rn   rp   s   &&&&&&&&&&&&&  €r   r   ÚOptimizer1State.__init__R  sò   ø€ ðX �bŒyÜÐ6°r°dÐ;Ó<Ð<Ø�cŒzÜÐ6°s°eÐ<Ó=Ð=Ü”s˜5“zÖ"ˆAØ˜%�(Ö( S×(Ð(Ü Ð#CÀAÀ3ÀbÈÍÈ
Ð!SÓTÐTñ #ð �lÔ"ÜÐ;¸L¸>ÐJÓKÐKÜ˜2°ÔOˆÜ‰Ñ˜¨:Ô@àŠ<ØˆDØ!+�ÑØ$1�Ñ!Ø )�ÑØ!+�Ñä  ›ˆD�IàŒIà,Ör   c                ór  € V P                  W4V4      pVR ,          ^ 8X  d   \        P                  pM4VR ,          ^8X  d   \        P                  pM\	        RVR ,           24      hVP                  4       VR,          8  d   \        P                  pV P                  V,          p^ VR&   V\        P                  8X  d'   V P                  V\        P                  R7      VR&   EMV\        P                  8X  Ed   VR,          ^ 8X  d[   RV P                  9  d   V P                  4        V P                  R,          P                  VP                  4      V P                  R&   V P                  V\        P                  R7      VR&   V P                  R,          VR&   RpVP                  4       p	W˜,          \        W˜,          4      ,           p
\        P                  ! V
3\        P                  VP                  R	7      VR
&   VR,          R8”  d(   \        P                  ! RVP                  R7      VR&   R# R# )ro   r   rÜ   rÑ   r!  r\   rt   rZ   r"  rí   r^   rÝ   rÙ   r#  rc   Nr$  r&  r)  s   &&&&&      r   rÌ   ÚOptimizer1State.init_state—  s©  € à—‘ °Ó7ˆà�,Õ 2Ô%Ü—M‘M‰EØ�LÕ! QÔ&Ü—K‘K‰Eä%Ð(PÐQWÐXdÕQeÐPfÐ&gÓhÐhà�7‰7‹9�v˜oÕ.Ô.Ü—M‘MˆEà—
‘
˜1•ˆØˆˆf‰à”E—M‘MÔ!Ø"×3Ñ3°A¼U¿]¹]Ð3ÓKˆE�(‹OØ”e—k‘kÕ!Ø�V�} Ô!Ø D§N¡NÔ2Ø—N‘NÔ$Ø,0¯N©N¸9Õ,E×,HÑ,HÈÏÉÓ,R�—‘˜yÑ)à"×3Ñ3°A¼U¿[¹[Ð3ÓIˆE�(‰OØ!Ÿ^™^¨IÕ6ˆE�'‰NàˆIØ—‘“	ˆAØ•n¬¨Q­]Ó(;Õ;ˆFä$Ÿ{š{¨F¨9¼E¿M¹MÐRS×RZÑRZÔ[ˆE�)Ñà�+Õ Ô$Ü!&§¢¨T¸!¿(¹(Ô!CˆE�+Óñ %r   c                óÜ  € VP                   P                  4       Vn         VP                  P                  4       Vn        V P                  V,          pVP                  pV P	                  W4V4      pVR ;;,          ^,          uu&   VR ,          pVR,          P
                  \        P                  8X  d‘   \        P                  ! V P                  TTVR,          VR,          ^ ,          VR,          TVR,          RVR,          ^,          RRVR,          RVR	,          R8”  d
   VR
,          MRVR	,          VR,          R7       R# VR,          P
                  \        P                  8X  dƒ   \        P                  ! V P                  VVVR,          RVR,          ^ ,          VR,          ^,          RRVR,          VVR,          VR,          RVR,          RVR,          RVR,          R7       R# R# )rÑ   r\   rÔ   rÕ   r×   NrÙ   rÖ   r  rÝ   rc   rÞ   r/  rZ   r^   r0  )r2  r3  rË   ry   rá   r¢   rA   r  rh   r4  r  r£   r5  r6  s	   &&&&&    r   rÎ   ÚOptimizer1State.update_step¼  s¡  € ð —‘×"Ñ"Ó$ˆŒØ—‘×"Ñ"Ó$ˆŒà—
‘
˜1•ˆØ�v‰vˆà—‘ °Ó7ˆàˆf�˜Õ‹Ø�V�}ˆà��?× Ñ ¤E§K¡KÔ/Ü×$Ò$Ø×#Ñ#ØØØ�h•Ø�w• Õ"Ø�u•ØØ�t•ØØ�w• Õ"ØØØ�~Õ&ØØ&,¨[Õ&9¸CÔ&?��kÖ"ÀTØ  Õ-Ø! ,Õ/÷#ð( �8�_×"Ñ"¤e§k¡kÔ1Ü×-Ò-Ø×#Ñ#ØØØ�h•ØØ�w• Õ"Ø�w• Õ"ØØØ�u•ØØ�t•Ø�g•ØØ�iÕ ØØ�~Õ&ØØ! ,Õ/÷'ñ 2r   r8  )
r9  )r:  rÙ   r;  rÙ   rý   Nr<  rÙ   FFr=  r  s   @@r   r?  r?  Q  sH   ù‡ € ÷C-ðJ ‡]‚]ƒ_ñ"Dó ð"DðH ‡]‚]ƒ_ñ7ó ÷7ð 7r   r?  )Úcollectionsr   r¦   r   Úcopyr   Ú	itertoolsr   ÚloggingÚtypingr   rî   rA   Úbitsandbytes.functionalÚ
functionalrh   Úbitsandbytes.utilsr   Ú	getLoggerr   Úloggerr
   r   ÚoptimÚ	OptimizerrU   r  r?  r   r   r   Ú<module>rR     s�   ð÷
 ;Ý Ý Û Ý Û ã å #Ý 'à	×	Ò	˜8Ó	$€÷2ñ 2÷XNñ XNôv[7�E—K‘K×)Ñ)ô [7ô|{�mô {ô|c�mö cr   