+
    œ°³jâ^  ã                  óæ  a € 0 t $ R t^ RIHt ^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHtHt ^ RIt^ RIHt ^ RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHt ^RIHtHtHtHtH t H!t!H"t"H#t#H$t$ ]'       d   ^RI%H&t& ^RIH't' ]PP                  ! ])4      t*]
 ! R R4      4       t+R R lt,]]+.]-]].]/]3,          3,          3,          t0R R lt1R;R R llt2R R lt3R R lt4R R lt5R R lt6R R  lt7R! R" lt8R# R$ lt9R% R& lt:R' R( lt;R) R* lt<R+ R, lt=R- R. lt>R/ R0 lt?R1 R2 lt@R3 R4 ltAR5 R6 ltBR7 R8 ltC]Pˆ                  ]PŠ                  ]PŒ                  ]PŽ                  ]P�                  ]P’                  ]P”                  ]P–                  ]P˜                  ]Pš                  ]Pœ                  ]Pž                  ]P                   ]P¢                  ]P¤                  .tS]P¨                  ]Pª                  ]P¬                  ]P®                  ]P°                  ]P²                  .tZ]P¶                  ]P¸                  .t]]P¼                  ]P¾                  .t`]PÂ                  ]PÄ                  ]PÆ                  .td]PÊ                  ]3]PÌ                  ]4]PÎ                  ]4]PÐ                  ]6]PÒ                  ]7]PÔ                  ]9]PÖ                  ]:]PØ                  ];]PÚ                  ]<]PÜ                  ]@]PÞ                  ]C]Pà                  ]?]Pâ                  ]?/].På                  ]S]84      C].På                  ]Z]=4      C].På                  ]]]>4      C].På                  ]`]A4      C].På                  ]d]B4      CtsR9]tR:&   R# )<z,
Optimizer utilities for the Trainer class.
)ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únn)Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óN   € ] tR t^6t$ RtR]R&   R]R&   R]R&   R]R&   R	]R
&   RtR# )ÚOptimizerContextz0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_args© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r"   ó    Ú_/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/trainer_optimizer.pyr   r   6   s#   ‡ á:à
ÓØ!Ó!Ø$Ó$ØÓØ×r*   r   c               ó    € V ^8„  d   QhRRRR/# )é   Úoptim_args_strz
str | NoneÚreturnr    r"   )Úformats   "r+   Ú__annotate__r1   A   s   € ÷ ñ  jð °^ñ r*   c                óœ   € V '       g   / # / pV P                  RR4      P                  R4       F  pVP                  R4      w  r4WAV&   K  	  V# )z8Parse optimizer arguments from a comma-separated string.Ú Ú Ú,Ú=)ÚreplaceÚsplit)r.   r!   ÚmappingÚkeyÚvalues   &    r+   Ú_parse_optim_argsr<   A   sP   € çØˆ	Ø€JØ!×)Ñ)¨#¨rÓ2×8Ñ8¸Ö=ˆØ—]‘] 3Ó'‰
ˆØ�3‹ñ >ð Ðr*   c               ó    € V ^8„  d   QhRRRR/# )r-   Úoptimizer_cls_or_factoryr   r/   Úboolr"   )r0   s   "r+   r1   r1   P   s   € ÷ ñ °3ð ¸4ñ r*   c                óŠ   € \        V \        4      '       d-   \        V \        P                  P
                  4      '       d   R# R# )a  
Check if the returned value from a handler is a factory rather than an Optimizer class.

Factory callables are used for complex optimizers like Muon or Dion that need to:
- Split parameters between multiple internal optimizers
- Handle complex sharding logic
- Access the full model structure for parameter grouping

Args:
    optimizer_cls_or_factory: The first element returned by an optimizer handler.

Returns:
    `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)r>   s   &r+   Úis_optimizer_factoryrG   P   s2   € ô  Ð*¬D×1Ò1´jÐAYÔ[`×[fÑ[f×[pÑ[p×6qÒ6qÙÙr*   c               ó8   € V ^8„  d   QhRRRRRRRRR	RR
RRRRR/# )r-   r   r   r   r   Úoptimizer_nameÚstrÚoptimizer_mappingr   Úoptim_kwargsr   Úis_layerwise_supportedr?   r/   útuple[Any, dict[str, Any]]r"   )r0   s   "r+   r1   r1   e   sb   € ÷ V+ñ V+Ø
ðV+àðV+ð ðV+ð &ð	V+ð
 !ðV+ð %ðV+ð !ðV+ð  ñV+r*   c                óV  a€ VP                  4       P                  R4      pV'       d7   V P                  \        P                  8X  d   V'       d   \        RV R24      hW2,          pV P                  f   \        RV R24      h\        V P                  \        \        34      '       g   \        RV P                   24      hVf   \        RV R24      h\        V P                  \        4      ;'       d     V P                  P                  R	R
4      R8H  p	. p
VP                  4        F–  w  r¼\        V P                  VRR7      w  rÞ\        V\        P                   4      '       g.   V'       d$   V'       g   \"        P%                  V RV R24       Kl  V'       g   V	'       g   K~  V
P'                  VR,           4       K˜  	  \)        V
4      ^ 8X  d   \        RV RV P                   R24      hVP+                  4        UUu. uF  w  ppWú9   g   K  VNK  	  pppVP+                  4        UUu. uF  w  ppWú9  g   K  VNK  	  pppRV/RV/VC.pV'       d²   V P,                  ^8w  d   \        RV R24      h/ oV F  pV! RV./.3/ VB SV&   K  	  V F  pV! RV./VC.3/ VB SV&   K  	  V3R lpVP/                  4        F(  pVP0                  '       g   K  VP3                  V4       K*  	  \4        pVP7                  RS/4       VP7                  RV/4       W…3# u uppi u uppi )zª
Helper function to set up low-rank optimizers like GaLore and Apollo.

These optimizers apply low-rank projections to specific target modules (typically linear layers).
Ú	layerwisezLayer-wise z" does not support DDP at this timez1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_Ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightzNo target modules found for z (z).Úparamsz
Layerwise z( does not support gradient accumulation!c                ó„   <€ V P                   e1   SV ,          P                  4        SV ,          P                  4        R # R # ©N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts   &€r+   Úoptimizer_hookÚ1_setup_low_rank_optimizer.<locals>.optimizer_hook®   s4   ø€ Ø�z‰zÒ%Ø˜uÕ%×*Ñ*Ô,Ø˜uÕ%×/Ñ/Ö1ñ &r*   r[   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorrA   ÚlistrJ   Ú	TypeErrorr7   Únamed_modulesr   r   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr
   Úupdate)r   r   rI   rK   rL   r   rM   Úis_layerwiseÚoptimizer_clsÚ
all_linearÚtarget_params_namesÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚnÚpÚtarget_paramsÚnon_target_paramsÚparam_groupsrZ   r\   r[   s   &&&&&&&               @r+   Ú_setup_low_rank_optimizerr€   e   s6  ø€ ð "×'Ñ'Ó)×2Ñ2°;Ó?€Lß˜×*Ñ*¬l×.FÑ.FÔF×KaÜ! K°Ð/?Ð?aÐ"bÓcÐcà%Õ5€Mà× Ñ Ò(ÜÐLÈ^ÐL\Ð\gÐhÓiÐiä�d×/Ñ/´$¼°×=Ò=ÜðØ×-Ñ-Ð.ð0ó
ð 	
ð
 ‚}ÜÐBÀ>ÐBRÐR]Ð^Ó_Ð_ô 	�4×,Ñ,¬cÓ2×rÐr°t×7PÑ7P×7XÑ7XÐY\Ð^aÓ7bÐfrÑ7rð ð ÐØ$×2Ñ2Ö4ÑˆÜ)CØ×%Ñ% {ÀDô*
Ñ&Ðô ˜&¤"§)¡)×,Ò,ß#¯HÜ—‘ + Ð.DÀ^ÐDTÐTqÐrÔsÙç#¯JÙà×"Ñ" ;°Õ#:Ö;ñ  5ô ÐÓ 1Ô$ÜÐ7¸Ð7GÀrÈ$×JcÑJcÐIdÐdfÐgÓhÐhà#(×#9Ñ#9Ô#;ÔXÑ#;™4˜1˜a¸qÑ?W—Q�QÑ#;€MÑXØ',×'=Ñ'=Ô'?Ô`Ñ'?™t˜q !À1ÑC_Ÿ˜Ñ'?ÐÑ`ð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€L÷
 Ø×+Ñ+¨qÔ0Ü˜z¨.Ð)9Ð9aÐbÓcÐcàˆÛ&ˆEÙ$1°H¸u¸gÐ3FÐ2GÑ$\ÐK[Ñ$\ˆN˜5Ó!ñ 'ã"ˆEÙ$1°H¸u¸gÐ3VÈÐ3VÐ2WÑ$lÐ[kÑ$lˆN˜5Ó!ñ #õ	2ð
 ×%Ñ%Ö'ˆEØ×"×"Ò"Ø×8Ñ8¸ÖHñ (ô 0ˆØ×ÑÐ!1°>Ð BÔCà×Ñ˜X |Ð4Ô5ØÐ*Ð*ùó? YùÛ`s   ÈLÈLÈ-L%È=L%c               ó    € V ^8„  d   QhRRRR/# ©r-   Úctxr   r/   rN   r"   )r0   s   "r+   r1   r1   Ã   s   € ÷ +ñ +Ð(ð +Ð-Gñ +r*   c                ód   € V P                   P                  RRRR/4       \        V P                   3# )zGet Adafactor optimizer.Úscale_parameterFÚrelative_step)r   rr   r	   )rƒ   s   &r+   Ú_get_adafactorr‡   Ã   s1   € à×Ñ×ÑÐ!2°E¸?ÈEÐ RÔSÜ�c×*Ñ*Ð*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   É   s   € ÷ 'ñ 'Ð*ð 'Ð/Iñ 'r*   c                ó¸  € ^ RI Hp V P                  P                  V P                  4       V P
                  P                  \        P                  8X  d   V P                  P                  RR/4       V P                  eP   \        R V P                  P                  4        4       4      '       d    V P                  P                  RRRR/4       WP                  3# )z/Get PyTorch AdamW optimizer (regular or fused).©ÚAdamWÚfusedTc              3  óL   "  € T F  qP                   '       g   K  Vx € K  	  R # 5irV   )rp   )Ú.0r|   s   & r+   Ú	<genexpr>Ú#_get_adamw_torch.<locals>.<genexpr>Ò   s   é € Ð2hÑ>T¸×XgÕXg·1²1Ó>Tùs   ‚$š
$FÚforeach)Útorch.optimr‹   r   rr   r   r   rE   r   ÚADAMW_TORCH_FUSEDr   Úhas_mixed_dtensorro   ©rƒ   r‹   s   & r+   Ú_get_adamw_torchr–   É   s¡   € å!à×Ñ×Ñ §¡Ô0Ø
‡x�x‡~�~œ×9Ñ9Ô9Ø×Ñ×#Ñ# W¨d OÔ4ð ‡y�yÒÔ!2Ñ2h¸c¿i¹i×>RÑ>RÔ>TÓ2h×!hÒ!hØ×Ñ×#Ñ# W¨e°YÀÐ$FÔGØ×&Ñ&Ð&Ð&r*   c               ó   € V ^8„  d   QhRR/# )r-   r/   r?   r"   )r0   s   "r+   r1   r1   ×   s   € ÷ 
ñ 
 $ñ 
r*   c                óN  a€ ^ RI Ho \        V 4      p \        V 4      ;'       d‚    \        ;QJ d    V3R lV  4       F  '       g   K   RM	  RM! V3R lV  4       4      ;'       d@    \
        ;QJ d    V3R lV  4       F  '       d   K   RM	  RM! V3R lV  4       4      '       * # )z‡Whether `tensors` mixes `DTensor` and plain tensors, as expert parallelism leaves a model:
experts sharded, everything else replicated.)ÚDTensorc              3  ó<   <"  € T F  p\        VS4      x € K  	  R # 5irV   ©rA   ©rŽ   Útr™   s   & €r+   r�   Ú$has_mixed_dtensor.<locals>.<genexpr>ß   s   øé € Ð8±¨1”
˜1˜g×&Ð&³ùó   ƒTFc              3  ó<   <"  € T F  p\        VS4      x € K  	  R # 5irV   r›   rœ   s   & €r+   r�   rž   à   s   øé € Ð<±G¨q”J˜q '×*Ð*³GùrŸ   )Útorch.distributed.tensorr™   re   r?   ÚanyÚall)Útensorsr™   s   &@r+   r”   r”   ×   sp   ø€ õ 1ä�7‹m€GäˆW‹÷ 	=ð 	=ß‹CÔ8±Ó8�C�CŠCÔ8±Ó8Ó8÷	=ð 	=ç“Ô<±GÓ<——’Ô<±GÓ<Ó<Ô<ðr*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ä   s    € ÷ Tñ TÐ.ð TÐ3Mñ Tr*   c                ó¨   €  ^ RI Hp V P                  P                  V P                  4       WP                  3#   \
         d    \        R4      hi ; i)z'Get Torch XLA syncfree AdamW optimizer.rŠ   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreer‹   r   rr   r   ÚImportErrorrd   r•   s   & r+   Ú_get_adamw_torch_xlar©   ä   sO   € ðTÝ0à×Ñ×#Ñ# C§O¡OÔ4Ø×*Ñ*Ð*Ð*øÜô TÜÐRÓSÐSðTúó	   ‚7: ºAc               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ï   s    € ÷ Pñ PÐ$4ð PÐ9Sñ Pr*   c                ó¨   €  ^ RI Hp V P                  P                  V P                  4       WP                  3#   \
         d    \        R4      hi ; i)zGet NPU Fused AdamW optimizer.)ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimr­   r   rr   r   r¨   rd   )rƒ   r­   s   & r+   Ú_get_adamw_torch_npu_fusedr¯   ï   sO   € ðPÝ1à×Ñ×#Ñ# C§O¡OÔ4Ø×2Ñ2Ð2Ð2øÜô PÜÐNÓOÐOðPúrª   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ú   s   € ÷ //ñ //Ð%5ð //Ð:Tñ //r*   c                óÚ  € \        4       '       g   \        R4      h^ RIHpHpHp V P                  P                  pRV9   pRV9   d   ^M^ pRpV P                  pRV9   d   TpEM¶RV9   d3   TpRV P                  P                  V P                  P                  3/pEM}R	V9   d   TpV P                  pEMfR
V9   Ed_   ^ RIHp	 T	pR\        V P                  P                  RV P                  P                  4      4      \        V P                  P                  RV P                  P                  4      4      \        V P                  P                  RR4      4      3R\        V P                  P                  RR4      4      R\        V P                  P                  RV P                  P                  4      4      /pRV P                  9   d    \!        V P                  R,          4      VR&   RV P                  9   d    \!        V P                  R,          4      VR&   RV/p
R	V9  d   WZR&   V P"                  P%                  V4       V P"                  P%                  V
4       WpP"                  3# )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`)r‹   ÚLionÚRMSpropÚpagedÚ8bitNÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @ÚepsÚt_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   r¨   Úbitsandbytes.optimr‹   r²   r³   r   rE   r   Ú
adam_beta1Ú
adam_beta2r!   r»   ÚfloatÚgetÚadam_epsilonÚintr   rr   )rƒ   r‹   r²   r³   Ú
optim_namerÄ   rÃ   rt   Úadditional_optim_kwargsr»   Ú
bnb_kwargss   &          r+   Ú_get_bitsandbytes_optimizerrÏ   ú   s  € ä$×&Ò&ÜØwó
ð 	
÷ 8Ñ7à—‘—‘€JØ˜*Ñ$€HØ 
Ô*‘°€JØ€MØ!Ÿo™oÐà�ÔØŠØ	�:Ô	ØˆØ#*¨S¯X©X×-@Ñ-@À#Ç(Á(×BUÑBUÐ,VÐ"WÒØ	�jÔ	 ØˆØ"%§.¡.ÒØ	�zÕ	!Ý/à ˆàÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°&Ó9Ó:ðð
 ”U˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ø”5˜Ÿ™×+Ñ+¨E°3·8±8×3HÑ3HÓIÓJð#
Ðð ˜Ÿ™Ô&Ü14°S·^±^ÀIÕ5NÓ1OÐ# IÑ.Ø˜Ÿ™Ô&Ü14°S·^±^ÀIÕ5NÓ1OÐ# IÑ.à 
Ð+€JØ˜
Ô"Ø!)�:Ñà×Ñ×ÑÐ 7Ô8Ø×Ñ×Ñ 
Ô+Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ,  s    € ÷ Qñ QÐ!1ð QÐ6Pñ Qr*   c                ó$  €  ^ RI Hp V P                  P                  V P                  4       V P                  P                  R\        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      R\        \        V P                  P                  RR4      4      /4       WP                  3#   \         d    \        R	4      hi ; i)
z!Get AnyPrecision AdamW optimizer.)ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersrÒ   r   rr   r   r   r!   rÉ   ÚgetattrrD   r¨   rd   )rƒ   rÒ   s   & r+   Ú_get_adamw_anyprecisionrÜ   ,  sæ   € ðQÝ;à×Ñ×#Ñ# C§O¡OÔ4Ø×Ñ×#Ñ#à%¤y°·±×1CÑ1CÐDYÐ[bÓ1cÓ'dØ ¤'¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aØ ¤'¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aØ+¬WÜ˜3Ÿ>™>×-Ñ-Ð.IÈ:ÓVó.ð	ô		
ð !×"6Ñ"6Ð6Ð6øÜô QÜÐOÓPÐPðQús   ‚C5C8 Ã8Dc               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   A  s   € ÷ 	#ñ 	#Ð"ð 	#Ð'Añ 	#r*   c                ót  € V P                   P                  4       pV P                  '       dr   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  RV P                  9   d(   V P                  R,          P	                  4       R9   VR&   \
        P                  P                  V3# )zGet SGD optimizer.Únesterov)ÚmomentumÚ	dampeningÚweight_decay©ÚtrueÚ1Úyes)r   Úcopyr!   rÈ   r^   rD   rE   ÚSGD©rƒ   Úkwargsr:   s   &  r+   Ú_get_sgdrë   A  s‹   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û<ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ =ð ˜Ÿ™Ô'Ø!$§¡°
Õ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;�?‰?˜FÐ"Ð"r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   M  s   € ÷ 'ñ 'Ð&ð 'Ð+Eñ 'r*   c                ó  € V P                   P                  4       pV P                  '       d:   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  \        P
                  P                  V3# )zGet Adagrad optimizer.)Úlr_decayrâ   rÀ   )r   rç   r!   rÈ   rD   rE   ÚAdagradré   s   &  r+   Ú_get_adagradrð   M  s_   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û6ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ 7ô �;‰;×Ñ Ð&Ð&r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   W  s   € ÷ 	'ñ 	'Ð&ð 	'Ð+Eñ 	'r*   c                ót  € V P                   P                  4       pV P                  '       dr   R F3  pW P                  9   g   K  \        V P                  V,          4      W&   K5  	  RV P                  9   d(   V P                  R,          P	                  4       R9   VR&   \
        P                  P                  V3# )zGet RMSprop optimizer.Úcentered)rà   r¿   rÀ   râ   rã   )r   rç   r!   rÈ   r^   rD   rE   r³   ré   s   &  r+   Ú_get_rmsproprô   W  sŽ   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‡~€~Û?ˆCØ—n‘nÖ$Ü# C§N¡N°3Õ$7Ó8�“ñ @ð ˜Ÿ™Ô'Ø!$§¡°
Õ!;×!AÑ!AÓ!CÐG[Ñ![ˆF�:ÑÜ�;‰;×Ñ Ð&Ð&r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   c  s   € ÷ +ñ +Ð/ð +Ð4Nñ +r*   c                ó2  € \        4       '       g   \        R4      h^ RIHpHpHp \        P                  V\        P                  V\        P                  V\        P                  V\        P                  V\        P                  V/pR\        V P                  P                  R^€4      4      R\        V P                  P                  R^È4      4      R\!        V P                  P                  RR4      4      RV P                  P                  RR4      /p\#        V P$                  V P&                  V P$                  P(                  WEV P*                  4      w  rgV P$                  P(                  \        P                  8X  d   VP-                  R	R
RR
/4       Wg3# )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`)ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚrankÚupdate_proj_gapÚscaleg      Ð?Ú	proj_typeÚstdr…   Fr†   )r   r¨   Úgalore_torchr÷   rø   rù   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISErË   r!   ÚpoprÈ   r€   r   r   rE   r   rr   )rƒ   r÷   rø   rù   rK   Úgalore_optim_kwargsrt   r   s   &       r+   Ú_get_galore_optimizerr  c  sQ  € ä$×&Ò&ÜðVó
ð 	
÷ KÑJô 	×#Ñ# [Ü×(Ñ(¨/Ü×'Ñ'¨Ü×-Ñ-¨{Ü×2Ñ2°OÜ×1Ñ1°?ðÐð 	”�C—N‘N×&Ñ& v¨sÓ3Ó4Øœ3˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJØ”�s—~‘~×)Ñ)¨'°4Ó8Ó9Ø�S—^‘^×'Ñ'¨°UÓ;ð	Ðô '@Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUió'Ñ#€Mð ‡x�x‡~�~œ×8Ñ8Ô8Ø×ÑÐ!2°E¸?ÈEÐ RÔSØÐ*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   „  s   € ÷ ñ Ð/ð Ð4Nñ r*   c                óÊ  € \        4       '       g   \        R4      h^ RIHp \        P
                  V\        P                  V/pR\        V P                  P                  R^€4      4      RV P                  P                  RR4      RV P                  P                  RR4      R\        V P                  P                  R^È4      4      R	\        V P                  P                  R	R
4      4      RV P                  P                  RR4      /pVP                  V P                  4       \        V P                  V P                  V P                  P                   W#V P"                  4      # )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`)ÚAPOLLOAdamWrú   ÚprojÚrandomÚ
scale_typeÚchannelrû   rü   ç      ð?rý   rþ   )r   r¨   Úapollo_torchr  r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISErË   r!   r  rÈ   rr   r   r€   r   r   rE   r   )rƒ   r  rK   Úapollo_optim_kwargss   &   r+   Ú_get_apollo_optimizerr  „  s%  € ä$×&Ò&ÜðUó
ð 	
õ )ô 	×#Ñ# [Ü×-Ñ-¨{ðÐð 	”�C—N‘N×&Ñ& v¨sÓ3Ó4Ø�—‘×"Ñ" 6¨8Ó4Ø�c—n‘n×(Ñ(¨°yÓAØœ3˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJØ”�s—~‘~×)Ñ)¨'°3Ó7Ó8Ø�S—^‘^×'Ñ'¨°UÓ;ðÐð ×Ñ˜sŸ™Ô/ä$Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->ÐUX×UiÑUióð r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ¡  s   € ÷ /ñ /Ð-ð /Ð2Lñ /r*   c                ó   € \        4       '       g   \        R4      hV P                  f   \        R4      h^ RIHpHp RV P                  P                  9   d   TMTpV P                  P                  RV P                  /4       W0P                  3# )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`zMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.)ÚAdaLomoÚLomoÚadar   )r   r¨   r   rd   Ú
lomo_optimr  r  r   rE   r   rr   )rƒ   r  r  rt   s   &   r+   Ú_get_lomo_optimizerr  ¡  sz   € ä×ÒÜð7ó
ð 	
ð
 ‡y�yÒÜÐhÓiÐiç(à$¨¯©¯©Ô6‘G¸D€MØ×Ñ×Ñ ¨#¯)©)Ð 4Ô5Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   ³  s   € ÷ +ñ +Ð(ð +Ð-Gñ +r*   c                ó  € \        4       '       g   \        R4      h^ RIHp V P                  P                  R\        V P                  P                  RR4      4      R\        V P                  P                  RR4      4      R\        V P                  P                  RR4      4      R	\        V P                  P                  R	R4      4      R
\        V P                  P                  R
R4      4      /4       WP                  3# )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`)Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingr  )	r   rd   Ú	grokadamwr  r   rr   rÈ   r!   rÉ   )rƒ   r  s   & r+   Ú_get_grokadamwr'  ³  sÎ   € ä!×#Ò#ÜÐPÓQÐQå#à×Ñ×Ñàœ% §¡× 2Ñ 2°<ÀÓ FÓGØ”E˜#Ÿ.™.×,Ñ,¨V°SÓ9Ó:Ø”U˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ø(¬%°·±×0BÑ0BÐC_ÐadÓ0eÓ*fØ¤ s§~¡~×'9Ñ'9Ð:MÈsÓ'SÓ!Tð	
ôð ×*Ñ*Ð*Ð*r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   Æ  s   € ÷  /ñ  /Ð 0ð  /Ð5Oñ  /r*   c                ó°  € \        4       '       dM   \        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8  d   \        R4      h\        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8:  d   \        R4      h\        P                  ! \        P                  P                  R4      4      \        P                  ! R4      8¼  d
   ^ RIHpHp M^ RI	HpHp V P                  P                  \        P                  8X  d   TpMTpV P                  P                  R	V P                   P#                  R	R
4      R\%        V P                   P#                  RR4      4      /4       V P                  P                  V P&                  4       W0P                  3# )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aorD   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0)Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundrÔ   )r   r   ÚparseÚ	importlibÚmetadatar¨   Útorchao.optimr+  r,  Útorchao.prototype.low_bit_optimr   rE   r   ÚADAMW_TORCH_4BITr   rr   r!   rÉ   r   r   )rƒ   r+  r,  rt   s   &   r+   Ú_get_torchao_optimizerr6  Æ  sU  € ä×!Ò!¤W§]¢]´9×3EÑ3E×3MÑ3MÈiÓ3XÓ%YÔ\c×\iÒ\iÐjqÓ\rÔ%rÜð,ó
ð 	
ô
 ‡}‚}”Y×'Ñ'×/Ñ/°Ó8Ó9¼W¿]º]È5Ó=QÔQÜð<ó
ð 	
ô ‡}‚}”Y×'Ñ'×/Ñ/°	Ó:Ó;¼w¿}º}ÈXÓ?VÔVß6Ð6çHà
‡x�x‡~�~œ×8Ñ8Ô8Ø!‰à!ˆà×Ñ×Ñà˜#Ÿ.™.×,Ñ,¨\¸3Ó?Ø#¤Y¨s¯~©~×/AÑ/AÐBYÐ[bÓ/cÓ%dð	
ôð ×Ñ×Ñ §¡Ô0Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1   é  s   € ÷ )/ñ )/Ð&6ð )/Ð;Uñ )/r*   c                ó`  € \        4       '       g   \        R4      h^ RIHpHp / pRpV P
                  P                  \        P                  8X  d4   \        R4      '       g   \        R4      h^ RIH	p TpV P                  pRpMoV P
                  P                  \        P                  8X  d   TpV P                  pM7V P
                  P                  \        P                  8X  d   TpM\        R4      hV P
                  P                  VR	&   V'       d   V P
                  P                  VR
&   VP!                  R\#        V P$                  P'                  RR4      4      R\#        V P$                  P'                  RR4      4      /4       V P(                  P!                  V4       W`P(                  3# )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`)ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerrâ   Úwarmup_stepsÚweight_lr_powerr"  Úrg        )r   r¨   Úschedulefreer9  r:  r   rE   r   ÚSCHEDULE_FREE_RADAMr;  r   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDrd   râ   r<  rr   rÈ   r!   rÉ   r   )rƒ   r9  r:  rÍ   Úrequire_warmupr;  rt   s   &      r+   Ú_get_schedule_free_optimizerrD  é  sc  € ä$×&Ò&Üð:ó
ð 	
÷ @à ÐØ€Nà
‡x�x‡~�~œ×;Ñ;Ô;Ü(¨×1Ò1Üð>óð õ 	3à)ˆØ"%§/¡/ÐØ‰Ø	�‰�‰œ>×=Ñ=Ô	=Ø)ˆØ"%§/¡/ÑØ	�‰�‰œ>×;Ñ;Ô	;Ø'‰äÐ9Ó:Ð:à.1¯h©h×.CÑ.CÐ˜NÑ+ßØ25·(±(×2GÑ2GÐ Ñ/Ø×"Ñ"àœu S§^¡^×%7Ñ%7Ð8IÈ3Ó%OÓPØ”�s—~‘~×)Ñ)¨#¨sÓ3Ó4ð	
ôð ×Ñ×ÑÐ 7Ô8Ø×.Ñ.Ð.Ð.r*   c               ó    € V ^8„  d   QhRRRR/# r‚   r"   )r0   s   "r+   r1   r1     s   € ÷ -ñ -Ð+ð -Ð0Jñ -r*   c                ó&  € \        4       '       g   \        R4      h^ RIHp V P                  P                  RR4      pVe   \        V4      pV P                  P                  RR4      pVe   \        V4      pV P                  P                  V P                  R&   R\        V P                  P                  RR4      4      RVRV/pV P                  P                  V P                  4       V P                  P                  V4       WP                  3# )	z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.)ÚStableAdamWÚmax_lrNÚ	kahan_sumrâ   Údecouple_lrF)r   r¨   ÚoptimirG  r!   r  rÈ   r?   r   râ   r   r   rr   )rƒ   rG  rH  rI  Ústable_adamw_kwargss   &    r+   Ú_get_stable_adamwrM    sï   € ä$×&Ò&Üð:ó
ð 	
õ #à�^‰^×Ñ ¨$Ó/€FØÒÜ�v“ˆà—‘×"Ñ" ;°Ó5€IØÒÜ˜“Oˆ	à&)§h¡h×&;Ñ&;€C‡O�O�NÑ#à”t˜CŸN™N×.Ñ.¨}¸eÓDÓEØ�&Ø�YðÐð ×Ñ×Ñ §¡Ô0Ø×Ñ×ÑÐ 3Ô4Ø×,Ñ,Ð,Ð,r*   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)T)uÚ__conditional_annotations__r'   Ú
__future__r   Úimportlib.metadatar1  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   rD   Ú	packagingr   r   Úoptimizationr	   Útrainer_pt_utilsr
   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr#   ri   r   r<   ÚtupleÚdictrJ   ÚOptimizerHandlerrG   r€   r‡   r–   r”   r©   r¯   rÏ   rÜ   rë   rð   rô   r  r  r  r'  r6  rD  rM  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSr   r  r  r  r  r  Ú_GALORE_OPTIMIZERSr  r  Ú_APOLLO_OPTIMIZERSr5  ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr@  rA  rB  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHr“   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_ANYPRECISIONrè   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysrN  r(   )rO  s   @r+   Ú<module>r‚     sW  øðôõ #ã Û Ý $Ý !ß %ã Ý Ý å #Ý 5Ý 5ß 7÷
÷ 
õ 
÷ Ý/Ý0à	×	Ò	˜8Ó	$€ð ÷ð ó ðõð Ð-Ð.°°c¸4ÀÀSÀ½>Ð6IÕ0JÐJÕKÐ õ÷*V+õ|+õ'õ
õTõPõ//õdQõ*	#õ'õ	'õ+õBõ:/õ$+õ& /õF)/õX-ðD ×ÑØ×ÑØ×ÑØ×#Ñ#Ø×ÑØ× Ñ Ø×!Ñ!Ø×&Ñ&Ø×ÑØ×ÑØ×ÑØ×"Ñ"Ø×ÑØ×ÑØ× Ñ ðÐ ð& ×ÑØ×$Ñ$Ø×#Ñ#Ø×)Ñ)Ø×.Ñ.Ø×-Ñ-ðÐ ð ×ÑØ×)Ñ)ðÐ ð ×#Ñ#Ø×#Ñ#ðÐ ð ×&Ñ&Ø×&Ñ&Ø×$Ñ$ðÐ ð ×Ñ˜nØ×ÑÐ 0Ø×$Ñ$Ð&6Ø×"Ñ"Ð$8Ø×(Ñ(Ð*DØ×%Ñ%Ð'>Ø×Ñ˜Ø×Ñ˜LØ×Ñ˜LØ×Ñ˜nØ×ÑÐ!2Ø×ÑÐ,Ø×ÑÐ/ð4ð ‡m�mÐ,Ð.IÓJð4ð ‡m�mÐ&Ð(=Ó>ð4ð  ‡m�mÐ&Ð(=Ó>ð!4ð" ‡m�mÐ'Ð)?Ó@ð#4ð$ ‡m�mÐ-Ð/KÓLð%4Ð Ð0õ r*   