+ jr^RIt^RIt^RIHuHuHuHt!RR]PP4t R#)Ncza]tRt^toRtV3RlRlt]R4tRtRt Rt Rt RV3R lR llt R t R tVtR #)PostLocalSGDOptimizera< Wraps an arbitrary :class:`torch.optim.Optimizer` and runs `post-local SGD `_, This optimizer runs local optimizer at every step. After the warm-up stage, it averages parameters periodically after the local optimizer is applied. Args: optim: The local optimizer. averager: A model averager instance to run post-localSGD algorithm. Example:: >>> # xdoctest: +SKIP("undefined variables") >>> import torch >>> import torch.distributed as dist >>> import torch.distributed.algorithms.model_averaging.averagers as averagers >>> import torch.nn as nn >>> from torch.distributed.optim import PostLocalSGDOptimizer >>> from torch.distributed.algorithms.ddp_comm_hooks.post_localSGD_hook import ( >>> PostLocalSGDState, >>> post_localSGD_hook, >>> ) >>> >>> model = nn.parallel.DistributedDataParallel( >>> module, device_ids=[rank], output_device=rank >>> ) >>> >>> # Register a post-localSGD communication hook. >>> state = PostLocalSGDState(process_group=None, subgroup=None, start_localSGD_iter=100) >>> model.register_comm_hook(state, post_localSGD_hook) >>> >>> # Create a post-localSGD optimizer that wraps a local optimizer. >>> # Note that ``warmup_steps`` used in ``PostLocalSGDOptimizer`` must be the same as >>> # ``start_localSGD_iter`` used in ``PostLocalSGDState``. >>> local_optim = torch.optim.SGD(params=model.parameters(), lr=0.01) >>> opt = PostLocalSGDOptimizer( >>> optim=local_optim, >>> averager=averagers.PeriodicModelAverager(period=4, warmup_steps=100) >>> ) >>> >>> # In the first 100 steps, DDP runs global gradient averaging at every step. >>> # After 100 steps, DDP runs gradient averaging within each subgroup (intra-node by default), >>> # and post-localSGD optimizer runs global model averaging every 4 steps after applying the local optimizer. >>> for step in range(0, 200): >>> opt.zero_grad() >>> loss = loss_fn(output, labels) >>> loss.backward() >>> opt.step() cb<V^8dQhRS[PPRS[P/#)optimaverager)torchr Optimizer averagers ModelAverager)format __classdict__s"p/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/distributed/optim/post_localSGD_optimizer.py __annotate__"PostLocalSGDOptimizer.__annotate__:s+!!ekk33!y?V?V!cTWnVPPVnW nR#N)r param_groupsr)selfrrs&&&r__init__PostLocalSGDOptimizer.__init__:s  JJ33 rc.VPP#r)rstaters&rrPostLocalSGDOptimizer.state?szzrc6VPP4#r)r__repr__rs&rrPostLocalSGDOptimizer.__repr__Cszz""$$rc lVPP4pVPPVR&V#)z This is the same as :class:`torch.optim.Optimizer` :meth:`state_dict`, but adds an extra entry to record model averager's step to the checkpoint to ensure reload does not cause unnecessary warm up again. step)r state_dictrr!)roptim_state_dicts& rr" PostLocalSGDOptimizer.state_dictFs2  ::002#'==#5#5 rc VPPV4RV9dVR,VPnR#\P !R^R7^VPnR#)a' This is the same as :class:`torch.optim.Optimizer` :meth:`load_state_dict`, but also restores model averager's step value to the one saved in the provided ``state_dict``. If there is no ``"step"`` entry in ``state_dict``, it will raise a warning and initialize the model averager's step to 0. r!z]Loaded state dict does not contain a step counter for an averager. Setting step counter to 0.) stacklevelN)rload_state_dictrr!warningswarn)rr"s&&rr'%PostLocalSGDOptimizer.load_state_dictPsQ "":. Z !+F!3DMM  MM-  "#DMM rc VPP4VPPVPR7R#)z9 Performs a single optimization step (parameter update). )paramsN)rr!raverage_parametersrrs&rr!PostLocalSGDOptimizer.stepds-  ((0A0A(Brc <V^8dQhRS[/#)r set_to_none)bool)r rs"rrrks66T6rc>VPPVR7R#))r0N)r zero_grad)rr0s&&rr3PostLocalSGDOptimizer.zero_gradks 5rc<VPPV4R#r)radd_param_group)r param_groups&&rr6%PostLocalSGDOptimizer.add_param_groupns "";/r)rrrN)T)__name__ __module__ __qualname____firstlineno____doc__rpropertyrrr"r'r!r3r6__static_attributes____classdictcell__)rs@rrrsR/b!!   % #(C6600rr) r(r 6torch.distributed.algorithms.model_averaging.averagers distributed algorithmsmodel_averagingr rr rrrrFs) JJg0EKK11g0r