+
     °³jÌ9  ã                   óú   € ^ RI t ^ RIt^ RIt^ RIHt ^ RIHt ^RIHt ^RI	H
t
 ^RIHt ^RIHt ^RIHt R	 tR
 t ! R R4      t ! R R4      t ! R R]4      t ! R R]
4      t ! R R4      t ! R R4      tR# )é    N)Údeepcopy)Úoptim)ÚAcceleratedOptimizer)ÚAcceleratedScheduler)ÚDistributedType)Úis_bnb_available)Úcompare_versionsc                óÀ  € V P                   P                  4        UUu/ uF  w  rVR9   g   K  WbK  	  ppp^ RIHp Tp\	        RRR4      '       d…   RVR&   \        V \        P                  4      p\        4       '       dI   V'       gA   ^ RI	Hp \        WP                  VP                  34      '       d    V P                  ^ 8H  pMRpV'       d   RVR&   \	        RRR	4      '       d€   \        V \        P                  4      p\        4       '       dF   V'       g>   ^ RI	Hp \        WP                  VP                  34      '       d    V P                  ^ 8H  pV'       d	   ^ R
IHp	 T	p\        RR7      '       dO   \	        RRR4      '       d<   ^ RI	Hp
Hp \        W
V34      '       d!    V P                  ^ 8H  pV'       d	   ^ RIHp TpV! V P,                  3/ VB # u uppi   \         d    T P                  P                  ^ 8H  p EL?i ; i  \         d    T P                  P                  ^ 8H  p LÚi ; i  \         d    T P                  P                  ^ 8H  p L¤i ; i)zx
Args:
    optimizer: torch.optim.Optimizer

Returns the DeepSeedCPUOptimizer (deepspeed.ops) version of the optimizer.
)ÚDeepSpeedCPUAdamÚ	deepspeedz>=z0.3.1FÚ
adamw_modeNTz0.5.5)ÚDeepSpeedCPUAdagradz0.38.0)Úmin_versionz0.11.0)ÚLionÚ	Lion32bit)ÚDeepSpeedCPULion)ÚlrÚweight_decay)ÚdefaultsÚitemsÚdeepspeed.ops.adamr   r	   Ú
isinstancer   ÚAdamWr   Úbitsandbytes.optimÚ
AdamW32bitÚ
optim_bitsÚAttributeErrorÚargsÚAdagradÚAdagrad32bitÚdeepspeed.ops.adagradr   r   r   Údeepspeed.ops.lionr   Úparam_groups)Ú	optimizerÚkÚvr   r   Úoptimizer_classÚis_adawÚbnb_optÚis_adar   r   r   Úis_bnb_32bitsr   s   &             Ú[/home/luca/screen-checking/.venv/lib/python3.14/site-packages/accelerate/utils/deepspeed.pyÚmap_pytorch_optim_to_deepspeedr-      s
  € ð "+×!3Ñ!3×!9Ñ!9Ô!;Ô[Ñ!;™˜¸qÐDZÑ?Z”�’Ñ!;€HÑ[õ 4à&€Oô ˜ T¨7×3Ò3Ø!&ˆ�ÑÜ˜Y¬¯©Ó4ˆä×Ò§gÝ0ä˜)§m¡m°W×5GÑ5GÐ%H×IÒIð>Ø'×2Ñ2°bÑ8‘Gð  �çØ%)ˆH�\Ñ"ô ˜ T¨7×3Ò3ä˜I¤u§}¡}Ó5ˆô ×Ò§fÝ0ä˜)§o¡o°w×7KÑ7KÐ%L×MÒMð=Ø&×1Ñ1°RÑ7�F÷ ÝAà1ˆOô  H×-Ó-Ô2BÀ;ÐPTÐV^×2_Ò2_ß6ä�i¨	Ð!2×3Ò3ð@Ø )× 4Ñ 4¸Ñ :�÷ Ý?à"2�á˜9×1Ñ1Ñ>°XÑ>Ð>ùóy \øô& &ô >Ø'Ÿn™n×7Ñ7¸2Ñ=“Gð>ûô( &ô =Ø&Ÿ^™^×6Ñ6¸"Ñ<’Fð=ûô "ô @Ø )§¡× 9Ñ 9¸RÑ ?’ð@ús@   žG®GÂ7G! Å	H Æ)H6 Ç!$H	ÈH	È$H3È2H3È6$IÉIc                ó  € V P                   \        P                  8w  d   \        R4      h\	        V P
                  \        4      '       g   V P
                  # \        R V P
                  P                  4        4       4      # )z‚
Returns the currently active DeepSpeedPlugin.

Raises:
    ValueError: If DeepSpeed was not enabled and this function is called.
a!  Couldn't retrieve the active `DeepSpeedPlugin` as none were enabled. Please make sure that either `Accelerator` is configured for `deepspeed` or make sure that the desired `DeepSpeedPlugin` has been enabled (`AcceleratorState().select_deepspeed_plugin(name)`) before calling this function.c              3   óL   "  € T F  qP                   '       g   K  Vx € K  	  R # 5i©N)Úselected)Ú.0Úplugins   & r,   Ú	<genexpr>Ú.get_active_deepspeed_plugin.<locals>.<genexpr>t   s   é € ÐYÑ%E˜6ÏÍ—’Ó%Eùs   ‚$š
$)	Údistributed_typer   Ú	DEEPSPEEDÚ
ValueErrorr   Údeepspeed_pluginsÚdictÚnextÚvalues)Ústates   &r,   Úget_active_deepspeed_pluginr>   d   sk   € ð ×Ñ¤×!:Ñ!:Ô:Üð,ó
ð 	
ô �e×-Ñ-¬t×4Ò4Ø×&Ñ&Ð&ÜÑY U×%<Ñ%<×%CÑ%CÔ%EÓYÓYÐYó    c                   óh   a € ] tR t^wt o RtR tR tR tRR ltRR lt	R t
R	 tR
 tR tR tRtV tR# )ÚHfDeepSpeedConfiga"  
This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage.

A `weakref` of this object is stored in the module's globals to be able to access the config from areas where
things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore
it's important that this object remains alive while the program is still running.

[`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration
with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic
the DeepSpeed configuration is not modified in any way.

Args:
    config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict.

c                ó~  € \        V\        4      '       d   \        V4      pMr\        P                  P                  V4      '       d6   \        VR R7      ;_uu_ 4       p\        P                  ! V4      pRRR4       M  \        P                  ! V4      pXV n        V P%                  4        R#   + '       g   i     L); i  \        P                   d?    \        P                  ! T4      P                  R 4      p\        P                  ! T4      p L~i ; i  \        \        \         3 d    \!        RT 24      hi ; i)zutf-8)ÚencodingNzoExpected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: )r   r:   r   ÚosÚpathÚexistsÚopenÚjsonÚloadÚloadsÚJSONDecodeErrorÚbase64Úurlsafe_b64decodeÚdecodeÚUnicodeDecodeErrorr   r8   ÚconfigÚset_stage_and_offload)ÚselfÚconfig_file_or_dictrP   ÚfÚconfig_decodeds   &&   r,   Ú__init__ÚHfDeepSpeedConfig.__init__ˆ   s  € ÜÐ)¬4×0Ò0ô Ð1Ó2‰FÜ�W‰W�^‰^Ð/×0Ò0ÜÐ)°G×<Õ<ÀÜŸš 1›�÷ =Ð<ðð8ä!ŸZšZÐ(;Ó<�Fð ˆŒà×"Ñ"Ö$÷% =×<ûô ×+Ñ+ô 8ä%+×%=Ò%=Ð>QÓ%R×%YÑ%YÐZaÓ%b�NÜ!ŸZšZ¨Ó7’Fð8ûô '¬¼
ÐCô Ü ð Fð  GZð  F[ð  \óð ðús1   ÁB.Á?C Â.B>	ÃADÄD ÄDÄD Ä%D<c                óL  € V P                  R R4      V n        RV n        V P                  4       '       g   V P	                  4       '       dZ   \        RR.4      p\        V P                  R4      V P                  R4      .4      p\        W!,          4      ^ 8”  d
   RV n        R# R# R# )	zzero_optimization.stageFÚcpuÚnvmez*zero_optimization.offload_optimizer.devicez&zero_optimization.offload_param.deviceTNéÿÿÿÿ)Ú	get_valueÚ_stageÚ_offloadÚis_zero2Úis_zero3ÚsetÚlen)rR   Úoffload_devices_validÚoffload_devicess   &  r,   rQ   Ú'HfDeepSpeedConfig.set_stage_and_offload¢   s‘   € ð —n‘nÐ%>ÀÓCˆŒð ˆŒØ�=‰=�?Š?˜dŸm™mŸošoÜ$'¨°¨Ó$8Ð!Ü!à—N‘NÐ#OÓPØ—N‘NÐ#KÓLðóˆOô �?Õ:Ó;¸aÔ?Ø $�–ñ @ñ .r?   c                óª   € V P                   pVP                  R 4      pVP                  4       pV F  pVP                  V4      pVe   K  RV3u # 	  W$3# )Ú.N)rP   ÚsplitÚpopÚget)rR   Úds_key_longrP   ÚnodesÚds_keyÚnodes   &&    r,   Úfind_config_nodeÚ"HfDeepSpeedConfig.find_config_nodeµ   sY   € Ø—‘ˆð ×!Ñ! #Ó&ˆØ—‘“ˆÛˆDØ—Z‘Z Ó%ˆFØŒ~Ø˜V�|Ò#ñ ð
 ˆ~Ðr?   Nc                óV   € V P                  V4      w  r4Vf   V# VP                  WB4      # )z7
Returns the set value or `default` if no value is set
)ro   rj   )rR   rk   ÚdefaultrP   rm   s   &&&  r,   r\   ÚHfDeepSpeedConfig.get_valueÂ   s/   € ð ×.Ñ.¨{Ó;‰ˆØŠ>ØˆNØ�z‰z˜&Ó*Ð*r?   c                óú   € V P                   pVP                  R4      pV F@  pTpVP                  V4      pVe   K  V'       d   \        RV RV P                    24      h R# 	  Xe   VP	                  X4       R# R# )z{
Deletes a sub-section of the config file if it's found.

Unless `must_exist` is `True` the section doesn't have to exist.
rg   NzCan't find z entry in the config: )rP   rh   rj   r8   ri   )rR   rk   Ú
must_existrP   rl   rn   Úparent_configs   &&&    r,   Údel_config_sub_treeÚ%HfDeepSpeedConfig.del_config_sub_treeË   sƒ   € ð —‘ˆð ×!Ñ! #Ó&ˆÛˆDØ"ˆMØ—Z‘Z Ó%ˆFØŒ~ßÜ$ {°;°-Ð?UÐVZ×VaÑVaÐUbÐ%cÓdÐdâñ ð Ò$Ø×Ñ˜dÖ#ñ %r?   c                óF   € V P                  V4      pVf   R# \        V4      # )zÖ
Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
specific question of whether the value is set to `True` (and it's not set to `False`` or isn't set).

F©r\   Úbool©rR   rk   Úvalues   && r,   Úis_trueÚHfDeepSpeedConfig.is_trueâ   s%   € ð —‘˜{Ó+ˆØšˆuÐ6¬4°«;Ð6r?   c                óP   € V P                  V4      pVf   R# \        V4      '       * # )zÕ
Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
specific question of whether the value is set to `False` (and it's not set to `True`` or isn't set).
Frz   r|   s   && r,   Úis_falseÚHfDeepSpeedConfig.is_falseë   s'   € ð
 —‘˜{Ó+ˆØšˆuÐ:¬t°E«{¬?Ð:r?   c                ó    € V P                   ^8H  # )é   ©r]   ©rR   s   &r,   r_   ÚHfDeepSpeedConfig.is_zero2ó   ó   € Ø�{‰{˜aÑÐr?   c                ó    € V P                   ^8H  # )é   r…   r†   s   &r,   r`   ÚHfDeepSpeedConfig.is_zero3ö   rˆ   r?   c                ó   € V P                   # r0   )r^   r†   s   &r,   Ú
is_offloadÚHfDeepSpeedConfig.is_offloadù   s   € Ø�}‰}Ðr?   )r^   r]   rP   r0   )F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rV   rQ   ro   r\   rw   r~   r�   r_   r`   r�   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r,   rA   rA   w   sA   ø‡ € ñò %ò4%ò&ô+ô$ò.7ò;ò ò ÷ð r?   rA   c                   ó:   a € ] tR t^ýt o RtR tRR ltR tRtV t	R# )ÚDeepSpeedEngineWrapperzÊ
Internal wrapper for deepspeed.runtime.engine.DeepSpeedEngine. This is used to follow conventional training loop.

Args:
    engine (deepspeed.runtime.engine.DeepSpeedEngine): deepspeed engine to wrap
c                ó   € Wn         R # r0   ©Úengine)rR   rœ   s   &&r,   rV   ÚDeepSpeedEngineWrapper.__init__  s   € ØŽr?   c                óÀ   € V P                   P                  VR 7       V P                   P                  ! V3/ VB  V'       d   V P                   P                  4        R# R# ))Úis_boundaryN)rœ   Ú"set_gradient_accumulation_boundaryÚbackwardÚstep)rR   ÚlossÚsync_gradientsÚkwargss   &&&,r,   r¡   ÚDeepSpeedEngineWrapper.backward  sN   € ð 	�‰×6Ñ6À>Ð6ÔRð 	�‰×Ò˜TÑ, VÒ,÷ ð �K‰K×ÑÖñ r?   c                ó~   € V P                   P                  4       p\        VR4      '       d   VP                  4       # V# )z3Get the global gradient norm from DeepSpeed engine.Úitem)rœ   Úget_global_grad_normÚhasattrr¨   )rR   Ú	grad_norms   & r,   r©   Ú+DeepSpeedEngineWrapper.get_global_grad_norm  s4   € à—K‘K×4Ñ4Ó6ˆ	ä�9˜f×%Ò%Ø—>‘>Ó#Ð#ØÐr?   r›   N)T)
r�   r�   r‘   r’   r“   rV   r¡   r©   r”   r•   r–   s   @r,   r™   r™   ý   s   ø‡ € ñòô÷,ð r?   r™   c                   óV   a a€ ] tR tRt oRtV 3R ltRR ltR t]R 4       t	Rt
VtV ;t# )	ÚDeepSpeedOptimizerWrapperi'  z‰
Internal wrapper around a deepspeed optimizer.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
c                ób   <€ \         SV `  VR RR7       \        V P                  R4      V n        R# )FN)Údevice_placementÚscalerÚoverflow)ÚsuperrV   rª   r$   Ú__has_overflow__)rR   r$   Ú	__class__s   &&€r,   rV   Ú"DeepSpeedOptimizerWrapper.__init__0  s*   ø€ Ü‰Ñ˜°UÀ4ÐÔHÜ '¨¯©¸
Ó CˆÖr?   c                ó   € R # r0   © )rR   Úset_to_nones   &&r,   Ú	zero_gradÚ#DeepSpeedOptimizerWrapper.zero_grad4  ó   € Ùr?   c                ó   € R # r0   r¸   r†   s   &r,   r¢   ÚDeepSpeedOptimizerWrapper.step7  r¼   r?   c                óV   € V P                   '       d   V P                  P                  # R# )zTWhether or not the optimizer step was done, or skipped because of gradient overflow.F)r´   r$   r²   r†   s   &r,   Ústep_was_skippedÚ*DeepSpeedOptimizerWrapper.step_was_skipped:  s#   € ð × × Ð Ø—>‘>×*Ñ*Ð*Ùr?   )r´   r0   )r�   r�   r‘   r’   r“   rV   rº   r¢   ÚpropertyrÀ   r”   r•   Ú__classcell__©rµ   r—   s   @@r,   r®   r®   '  s3   ù‡ € ñõDôòð ñó ÷ð r?   r®   c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚDeepSpeedSchedulerWrapperiB  zÆ
Internal wrapper around a deepspeed scheduler.

Args:
    scheduler (`torch.optim.lr_scheduler.LambdaLR`):
        The scheduler to wrap.
    optimizers (one or a list of `torch.optim.Optimizer`):
c                ó&   <€ \         SV `  W4       R # r0   )r³   rV   )rR   Ú	schedulerÚ
optimizersrµ   s   &&&€r,   rV   Ú"DeepSpeedSchedulerWrapper.__init__L  s   ø€ Ü‰Ñ˜Ö/r?   c                ó   € R # r0   r¸   r†   s   &r,   r¢   ÚDeepSpeedSchedulerWrapper.stepO  r¼   r?   r¸   )
r�   r�   r‘   r’   r“   rV   r¢   r”   r•   rÃ   rÄ   s   @@r,   rÆ   rÆ   B  s   ù‡ € ñõ0÷ò r?   rÆ   c                   ó.   a € ] tR tRt o RtRR ltRtV tR# )Ú
DummyOptimiS  aÎ  
Dummy optimizer presents model parameters or param groups, this is primarily used to follow conventional training
loop when optimizer config is specified in the deepspeed config file.

Args:
    lr (float):
        Learning rate.
    params (iterable): iterable of parameters to optimize or dicts defining
        parameter groups
    weight_decay (float):
        Weight decay.
    **kwargs (additional keyword arguments, *optional*):
        Other arguments.
c                ó6   € Wn         W n        W0n        W@n        R # r0   )Úparamsr   r   r¥   )rR   rÐ   r   r   r¥   s   &&&&,r,   rV   ÚDummyOptim.__init__c  s   € ØŒØŒØ(ÔØŽr?   )r¥   r   rÐ   r   N)gü©ñÒMbP?r   ©r�   r�   r‘   r’   r“   rV   r”   r•   r–   s   @r,   rÎ   rÎ   S  s   ø‡ € ñ÷ò r?   rÎ   c                   ó.   a € ] tR tRt o RtRR ltRtV tR# )ÚDummySchedulerij  aŒ  
Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
loop when scheduler config is specified in the deepspeed config file.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
    total_num_steps (int, *optional*):
        Total number of steps.
    warmup_num_steps (int, *optional*):
        Number of steps for warmup.
    lr_scheduler_callable (callable, *optional*):
        A callable function that creates an LR Scheduler. It accepts only one argument `optimizer`.
    **kwargs (additional keyword arguments, *optional*):
        Other arguments.
Nc                óB   € Wn         W n        W0n        W@n        WPn        R # r0   )r$   Útotal_num_stepsÚwarmup_num_stepsÚlr_scheduler_callabler¥   )rR   r$   rÖ   r×   rØ   r¥   s   &&&&&,r,   rV   ÚDummyScheduler.__init__|  s   € Ø"ŒØ.ÔØ 0ÔØ%:Ô"ØŽr?   )r¥   rØ   r$   rÖ   r×   )Nr   NrÒ   r–   s   @r,   rÔ   rÔ   j  s   ø‡ € ñ÷"ò r?   rÔ   )rL   rH   rD   Úcopyr   Útorchr   r$   r   rÈ   r   Údataclassesr   Úimportsr   Úversionsr	   r-   r>   rA   r™   r®   rÆ   rÎ   rÔ   r¸   r?   r,   Ú<module>rß      sx   ðó Û Û 	Ý å å ,Ý ,Ý (Ý %Ý &òD?òNZ÷&Cñ C÷L'ñ 'ôTÐ 4ô ô6Ð 4ô ÷"ñ ÷.ó r?   