+ jԠZ^RIHt^RIt^RIHt^RIHtHt^RIH t H t H t ^RI H t^RIHt^RIHt.ROt!RR]4t!RR]4t!RR]]4t!RR ]4t!RR ]]4t!RR ]4t!RR ]]4t!RR ]4t!RR]]4t!RR]4t R#))AnyNTensor) functionalinit) ParameterUninitializedBufferUninitializedParameter) SyncBatchNorm)LazyModuleMixin)Module BatchNorm1dLazyBatchNorm1d BatchNorm2dLazyBatchNorm2d BatchNorm3dLazyBatchNorm3dr caa]tRt^toRt^t.ROtRRR/V3RlV3RlllltV3RlRltV3RlR lt R t R t V3R lV3R llt V3Rlt RtVtV;t#) _NormBasez,Common base of _InstanceNorm and _BatchNorm.TbiascP<V^8dQhRS[RS[RS[R,RS[RS[RS[RR/#  num_featuresepsmomentumNaffinetrack_running_statsrreturnintfloatbool)format __classdict__s"[/home/luca/screen-checking/.venv/lib/python3.14/site-packages/torch/nn/modules/batchnorm.py __annotate___NormBase.__annotate__&sU3 3 3 3 $, 3  3 " 3 3  3 c <RVRV/p \S V`4WnW nW0nW@nWPnVP 'dg\\P!V3/V B4Vn V'd'\\P!V3/V B4Vn M7VPRR4M$VPRR4VPRR4VP 'dVPR\P!V3/V B4VPR\P!V3/V B4VVTPR\P !RR\P"/V P%4U U u/uFwrV R8wgKWbK up p B4VM6VPRR4VPRR4VPRR4VP'4R#uup p i) devicedtyperNweight running_mean running_varnum_batches_trackedr)super__init__rrrrrrtorchemptyr-rregister_parameterregister_bufferzerosonestensorlongitemsreset_parameters) selfrrrrrr+r,rfactory_kwargskv __class__s &&&&&&&&$ r&r3_NormBase.__init__&s#FGU; (  #6 ;;;#EKK $O$OPDK%ekk,&Q.&QR ''5  # #Hd 3  # #FD 1  # # #   L KN K   uzz,I.I     % **)7(<(<(>O(>!w,tqt(>O      6   5  !6 = Ps 5 G G c<V^8dQhRR/#rrN)r$r%s"r&r'r([s--T-r)cVP'dRVPP4VPP ^4VP P4R#R#)N)rr.zero_r/fill_r0r>s&r&reset_running_stats_NormBase.reset_running_stats[sM  # # #    # # %    " "1 %  $ $ * * , $r)c<V^8dQhRR/#rErF)r$r%s"r&r'r(cs''$'r)cVP4VP'dS\P!VP4VP e#\P !VP 4R#R#R#N)rLrrones_r-rzeros_rKs&r&r=_NormBase.reset_parameterscsK   " ;;; JJt{{ #yy$ DII&% r)c\hrP)NotImplementedErrorr>inputs&&r&_check_input_dim_NormBase._check_input_dimjs!!r)cZRP!R/VPBRVPRJ/B#)z{{num_features}, eps={eps}, momentum={momentum}, affine={affine}, bias={use_bias}, track_running_stats={track_running_stats}use_biasNrF)r$__dict__rrKs&r& extra_repr_NormBase.extra_reprms: IIO P -- *.))4*?  r)c<V^8dQhRR/#rErF)r$r%s"r&r'r(us      r)c <VPRR4pVeV^8dVP'dVR,p W9drVPe<VPP\P!R48wd VPM%\P !^\P R7W&\S V`!VVVVVVV4R#)versionNr0meta)r,) getrr0r+r4r:r;r2_load_from_state_dict) r> state_dictprefixlocal_metadatastrict missing_keysunexpected_keys error_msgsranum_batches_tracked_keyrBs &&&&&&&& r&rd_NormBase._load_from_state_dictus!$$Y5 Ow{0H0H0H'-/D&D #&8//;00775<<;OO,,auzz: 3 %        r)cd<V^8dQh/S[;R&S[;R&S[R,;R&S[;R&S[;R&#)rrrNrrrr )r$r%s"r&r'r(sH   J dl Lr))rrrrrrr-)rrrrrh㈵>皙?TTNN)__name__ __module__ __qualname____firstlineno____doc___version __constants__r3rLr=rXr]rd__annotate_func____static_attributes____classdictcell__ __classcell__rBr%s@@r&rrsW6HXM3 3 3 j--''"    yr)rc\aa]tRt^toRRR/V3RlV3RlllltV3RlRltRtVtV;t#) _BatchNormTrcP<V^8dQhRS[RS[RS[R,RS[RS[RS[RR/#rr )r$r%s"r&r'_BatchNorm.__annotate__sU    $,    "     r)cF<RVRV/p \S V`!VVVVV3/V BRV/BR#r+r,rN)r2r3) r>rrrrrr+r,rr?rBs &&&&&&&&$ r&r3_BatchNorm.__init__sC#FGU;           r)c&<V^8dQhRS[RS[/#rrWrr)r$r%s"r&r'rs0 0 V0 0 r)c VPV4VPfRpM VPpVP'dsVP'daVPeSVPP ^4VPfR\ VP4, pM VPpVP'dRpM$VPRJ;'dVPRJp\P!TVP'dVP'd VPMRVP'dVP'd VPMRVPVPVVVP4#)N?T)rXrtrainingrr0add_r"r.r/F batch_normr-rr)r>rWexponential_average_factor bn_trainings&& r&forward_BatchNorm.forwards; e$ == ), &)- & ===T555''3((--a0==(14uT=U=U7V1V.15.  ===K,,4TT4;K;Kt;SK || }}}(@(@(@!!$(MMMT5M5M5MD  SW KK II  & HH  r)rFro) rrrsrtrur3rrzr{r|r}s@@r&rrs(   .0 0 0 r)rcaa]tRt^toR RR/V3RlV3RlllltV3RlV3RlltV3RlRltV3R ltR tVt V;t #) _LazyNormBaseTrc<V^8dQhRR/#rErF)r$r%s"r&r'_LazyNormBase.__annotate__s** *r)c <RVRV/p\S V`!^VVRR3/VBRR/BW0nW@nVP'd)\ R/VBVnV'd\ R/VBVnVP'dy\R/VBVn\R/VBVn \P!RR\P/VP4U U u/uFwrV R8wgKWbK up p BVnR#R#uup p i)r+r,FrNrFr1)r2r3rrr r-rr r.r/r4r:r;r<r0) r>rrrrr+r,rr?r@rArBs &&&&&&&$ r&r3_LazyNormBase.__init__s#FGU;          #6 ;;;0B>BDK2D^D  # # # 3 En ED 2D^DD ',||(jj(%3$8$8$:K$:DAa7l414$:K (D $ $Ls  C3C3c<V^8dQhRR/#rErF)r$r%s"r&r'rs''$'r)cz<VP4'g$VP^8wd\SV` 4R#R#R#)rN)has_uninitialized_paramsrr2r=)r>rBs&r&r=_LazyNormBase.reset_parameterss3,,..43D3D3I G $ &4J.r)c<V^8dQhRR/#rErF)r$r%s"r&r'rs$$d$r)cVP4'EdKVP^,VnVP'd\ VP \ 4'g \R4hVP PVP34VPeR\ VP\ 4'g \R4hVPPVP34VP'dMVPPVP34VPPVP34VP4R#R#)rHz-self.weight must be an UninitializedParameterNz+self.bias must be an UninitializedParameter)rshaperr isinstancer-r AssertionError materializerrr.r/r=rVs&&r&initialize_parameters#_LazyNormBase.initialize_parameterss  ( ( * * % AD {{{!$++/EFF(G ''):):(<=99(%dii1GHH,III))4+<+<*>?'''!!--&&(  ,,&&(  ! ! #+ +r)c2<V^8dQh/S[;R&S[;R&#)rr-r)r )r$r%s"r&r'rs ""  r))rrr0rr.r/rr-ro) rrrsrtrur3r=rryrzr{r|r}s@@r&rrs6***X'' $$kr)rc6a]tRtRtoRtV3RlRltRtVtR#)ri2a Applies Batch Normalization over a 2D or 3D input. Method described in the paper `Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift `__ . .. math:: y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension over the mini-batches and :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the number of features or channels of the input). By default, the elements of :math:`\gamma` are set to 1 and the elements of :math:`\beta` are set to 0. At train time in the forward pass, the variance is calculated via the biased estimator, equivalent to ``torch.var(input, correction=0)``. However, the value stored in the moving average of the variance is calculated via the unbiased estimator, equivalent to ``torch.var(input, correction=1)``. Also by default, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. If :attr:`track_running_stats` is set to ``False``, this layer then does not keep running estimates, and batch statistics are instead used during evaluation time as well. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. Because the Batch Normalization is done over the `C` dimension, computing statistics on `(N, L)` slices, it's common terminology to call this Temporal Batch Normalization. Args: num_features: number of features or channels :math:`C` of the input eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` Shape: - Input: :math:`(N, C)` or :math:`(N, C, L)`, where :math:`N` is the batch size, :math:`C` is the number of features or channels, and :math:`L` is the sequence length - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) Examples:: >>> # With Learnable Parameters >>> m = nn.BatchNorm1d(100) >>> # Without Learnable Parameters >>> m = nn.BatchNorm1d(100, affine=False) >>> input = torch.randn(20, 100) >>> output = m(input) c<V^8dQhRR/#rErF)r$r%s"r&r'BatchNorm1d.__annotate__{TTTr)cVP4^8wd5VP4^8wd\RVP4 R24hR#R#rzexpected 2D or 3D input (got D input)Ndim ValueErrorrVs&&r&rXBatchNorm1d._check_input_dim{@ 99;!  q 0`__ . .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension over the mini-batches and :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the input size). By default, the elements of :math:`\gamma` are set to 1 and the elements of :math:`\beta` are set to 0. At train time in the forward pass, the standard-deviation is calculated via the biased estimator, equivalent to ``torch.var(input, correction=0)``. However, the value stored in the moving average of the standard-deviation is calculated via the unbiased estimator, equivalent to ``torch.var(input, correction=1)``. Also by default, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. If :attr:`track_running_stats` is set to ``False``, this layer then does not keep running estimates, and batch statistics are instead used during evaluation time as well. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. Because the Batch Normalization is done over the `C` dimension, computing statistics on `(N, H, W)` slices, it's common terminology to call this Spatial Batch Normalization. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` Shape: - Input: :math:`(N, C, H, W)` - Output: :math:`(N, C, H, W)` (same shape as input) Examples:: >>> # With Learnable Parameters >>> m = nn.BatchNorm2d(100) >>> # Without Learnable Parameters >>> m = nn.BatchNorm2d(100, affine=False) >>> input = torch.randn(20, 100, 35, 45) >>> output = m(input) c<V^8dQhRR/#rErF)r$r%s"r&r'BatchNorm2d.__annotate__NNNr)cjVP4^8wd\RVP4 R24hR#zexpected 4D input (got rNrrVs&&r&rXBatchNorm2d._check_input_dim0 99;! 6uyy{m8LM M r)rFNrrs@r&rrGRNNr)c:a]tRtRtoRt]tV3RlRltRtVt R#)riaA :class:`torch.nn.BatchNorm2d` module with lazy initialization. Lazy initialization is done for the ``num_features`` argument of the :class:`BatchNorm2d` that is inferred from the ``input.size(1)``. The attributes that will be lazily initialized are `weight`, `bias`, `running_mean` and `running_var`. Check the :class:`torch.nn.modules.lazy.LazyModuleMixin` for further documentation on lazy modules and their limitations. Args: eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` c<V^8dQhRR/#rErF)r$r%s"r&r'LazyBatchNorm2d.__annotate__rr)cjVP4^8wd\RVP4 R24hR#rrrVs&&r&rX LazyBatchNorm2d._check_input_dimrr)rFN) rrrsrtrurvrrrXrzr{rs@r&rr8 MNNr)c6a]tRtRtoRtV3RlRltRtVtR#)ria{ Applies Batch Normalization over a 5D input. 5D is a mini-batch of 3D inputs with additional channel dimension as described in the paper `Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift `__ . .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension over the mini-batches and :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the input size). By default, the elements of :math:`\gamma` are set to 1 and the elements of :math:`\beta` are set to 0. At train time in the forward pass, the standard-deviation is calculated via the biased estimator, equivalent to ``torch.var(input, correction=0)``. However, the value stored in the moving average of the standard-deviation is calculated via the unbiased estimator, equivalent to ``torch.var(input, correction=1)``. Also by default, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. If :attr:`track_running_stats` is set to ``False``, this layer then does not keep running estimates, and batch statistics are instead used during evaluation time as well. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. Because the Batch Normalization is done over the `C` dimension, computing statistics on `(N, D, H, W)` slices, it's common terminology to call this Volumetric Batch Normalization or Spatio-temporal Batch Normalization. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, D, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` Shape: - Input: :math:`(N, C, D, H, W)` - Output: :math:`(N, C, D, H, W)` (same shape as input) Examples:: >>> # With Learnable Parameters >>> m = nn.BatchNorm3d(100) >>> # Without Learnable Parameters >>> m = nn.BatchNorm3d(100, affine=False) >>> input = torch.randn(20, 100, 35, 45, 10) >>> output = m(input) c<V^8dQhRR/#rErF)r$r%s"r&r'BatchNorm3d.__annotate__arr)cjVP4^8wd\RVP4 R24hR#zexpected 5D input (got rNrrVs&&r&rXBatchNorm3d._check_input_dimarr)rFNrrs@r&rrrr)c:a]tRtRtoRt]tV3RlRltRtVt R#)rifaA :class:`torch.nn.BatchNorm3d` module with lazy initialization. Lazy initialization is done for the ``num_features`` argument of the :class:`BatchNorm3d` that is inferred from the ``input.size(1)``. The attributes that will be lazily initialized are `weight`, `bias`, `running_mean` and `running_var`. Check the :class:`torch.nn.modules.lazy.LazyModuleMixin` for further documentation on lazy modules and their limitations. Args: eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` c<V^8dQhRR/#rErF)r$r%s"r&r'LazyBatchNorm3d.__annotate__rr)cjVP4^8wd\RVP4 R24hR#rrrVs&&r&rX LazyBatchNorm3d._check_input_dimrr)rFN) rrrsrtrurvrrrXrzr{rs@r&rrfrr)caa]tRtRtoRtRRR/V3RlV3RlllltV3RlRltV3R lR ltV3R lR lt] RR l4t Rt Vt V;t #)r iaMApplies Batch Normalization over a N-Dimensional input. The N-D input is a mini-batch of [N-2]D inputs with additional channel dimension as described in the paper `Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift `__ . .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension over all mini-batches of the same process groups. :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the input size). By default, the elements of :math:`\gamma` are sampled from :math:`\mathcal{U}(0, 1)` and the elements of :math:`\beta` are set to 0. The standard-deviation is calculated via the biased estimator, equivalent to `torch.var(input, correction=0)`. Also by default, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. If :attr:`track_running_stats` is set to ``False``, this layer then does not keep running estimates, and batch statistics are instead used during evaluation time as well. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. Because the Batch Normalization is done for each channel in the ``C`` dimension, computing statistics on ``(N, +)`` slices, it's common terminology to call this Volumetric Batch Normalization or Spatio-temporal Batch Normalization. Currently :class:`SyncBatchNorm` only supports :class:`~torch.nn.DistributedDataParallel` (DDP) with single GPU per process. Use :meth:`torch.nn.SyncBatchNorm.convert_sync_batchnorm()` to convert :attr:`BatchNorm*D` layer to :class:`SyncBatchNorm` before wrapping Network with DDP. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, +)` eps: a value added to the denominator for numerical stability. Default: ``1e-5`` momentum: the value used for the running_mean and running_var computation. Can be set to ``None`` for cumulative moving average (i.e. simple average). Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters. Default: ``True`` track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics, and initializes statistics buffers :attr:`running_mean` and :attr:`running_var` as ``None``. When these buffers are ``None``, this module always uses batch statistics. in both training and eval modes. Default: ``True`` process_group: synchronization of stats happen within each process group individually. Default behavior is synchronization across the whole world bias: If set to ``False``, the layer will not learn an additive bias (only relevant if :attr:`affine` is ``True``). Default: ``True`` Shape: - Input: :math:`(N, C, +)` - Output: :math:`(N, C, +)` (same shape as input) .. note:: Synchronization of batchnorm statistics occurs only while training, i.e. synchronization is disabled when ``model.eval()`` is set or if ``self.training`` is otherwise ``False``. Examples:: >>> # xdoctest: +SKIP >>> # With Learnable Parameters >>> m = nn.SyncBatchNorm(100) >>> # creating process group (optional) >>> # ranks is a list of int identifying rank ids. >>> ranks = list(range(8)) >>> r1, r2 = ranks[:4], ranks[4:] >>> # Note: every rank calls into new_group for every >>> # process group created, even if that rank is not >>> # part of the group. >>> process_groups = [torch.distributed.new_group(pids) for pids in [r1, r2]] >>> process_group = process_groups[0 if dist.get_rank() <= 3 else 1] >>> # Without Learnable Parameters >>> m = nn.BatchNorm3d(100, affine=False, process_group=process_group) >>> input = torch.randn(20, 100, 35, 45, 10) >>> output = m(input) >>> # network is nn.BatchNorm layer >>> sync_bn_network = nn.SyncBatchNorm.convert_sync_batchnorm(network, process_group) >>> # only single gpu per process is currently supported >>> ddp_sync_bn_network = torch.nn.parallel.DistributedDataParallel( >>> sync_bn_network, >>> device_ids=[args.local_rank], >>> output_device=args.local_rank) Trcd<V^8dQhRS[RS[RS[R,RS[RS[RS[R,RS[R R/#) rrrrNrr process_grouprr)r!r"r#r)r$r%s"r&r'SyncBatchNorm.__annotate__sc++++$, +  + " +Tz++ +r)c R<RVRV/p \S V`!VVVVV3/V BRV /BW`nR#r)r2r3r) r>rrrrrrr+r,rr?rBs &&&&&&&&&$ r&r3SyncBatchNorm.__init__sK#FGU;           +r)c<V^8dQhRR/#rErF)r$r%s"r&r'r sWWWr)cjVP4^8d\RVP4 R24hR#)rz expected at least 2D input (got rNrrVs&&r&rXSyncBatchNorm._check_input_dim s/ 99;?? }HUV V r)c<V^8dQhRR/#rErF)r$r%s"r&r'rstr)cHVP^4^8Xd \R4hR#)rHz9SyncBatchNorm number of input channels should be non-zeroN)sizerrVs&&r&_check_non_zero_input_channels,SyncBatchNorm._check_non_zero_input_channelss' ::a=A K  r)c&<V^8dQhRS[RS[/#rr)r$r%s"r&r'rsaaVaar)c  tVPV4VPV4VPfRpM VPpVP'dVP'dqVP f \ R4hVP P^4VPf#RVP P4, pM VPpVP'dRpM$VPRJ;'dVPRJpVP'dVP'd VPMRpVP'dVP'd VPMRpT;'dYVP;'dE\PP4;'d\PP4pV'dVPP RRR\P"P%439d+\'R \P"P%4 24h\PP(P*pVP,'d VP,p\PP/V4pV^8pV'g<\0P2!VVVVP4VP6VVVP84#V'g \ R 4h\:P<!VVP4VP6VVVP8VXX4 #) z Runs the forward pass. Nrz$num_batches_tracked must not be NonerTcudahpuxpuz;SyncBatchNorm expected input tensor to be on GPU or XPU or zbn_training must be True)rXrrrrr0rritemr.r/r4 distributed is_availableis_initializedr+type_C_get_privateuse1_backend_namergroupWORLDrget_world_sizerrr-rrsync_batch_normapply) r>rWrrr.r/ need_syncr world_sizes && r&rSyncBatchNorm.forwards e$ ++E2 == ), &)- & ===T555''/$%KLL  $ $ ) )! ,}}$-043K3K3P3P3R-R*-1]]*  ===K,,4TT4;K;Kt;SK &*]]]d6N6N6ND  TX %)MMMT5M5M5MD  SW   3 3  3 3!!..0 3 3!!002  ||  668 ) !Qxx==?@B "--3399M!!! $ 2 2 **99-HJ"QI<<  *  $%?@@"((  *  r)c  lTp\V\PPPP 4'Ed&\PP VPVPVPVPVPVVPRJR7pVP'dI\P!4;_uu_4VPVnVPVn RRR4VPVnVP VnVP"VnVP$Vn\'VR4'dVP(VnVP+4F%wrEVP-W@P/WR44K' ?V# +'giL;i)aConverts all :attr:`BatchNorm*D` layers in the model to :class:`torch.nn.SyncBatchNorm` layers. Args: module (nn.Module): module containing one or more :attr:`BatchNorm*D` layers process_group (optional): process group to scope synchronization, default is the whole world Returns: The original :attr:`module` with the converted :class:`torch.nn.SyncBatchNorm` layers. If the original :attr:`module` is a :attr:`BatchNorm*D` layer, a new :class:`torch.nn.SyncBatchNorm` layer object will be returned instead. Example:: >>> # Network with nn.BatchNorm layer >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) >>> module = torch.nn.Sequential( >>> torch.nn.Linear(20, 100), >>> torch.nn.BatchNorm1d(100), >>> ).cuda() >>> # creating process group (optional) >>> # ranks is a list of int identifying rank ids. >>> ranks = list(range(8)) >>> r1, r2 = ranks[:4], ranks[4:] >>> # Note: every rank calls into new_group for every >>> # process group created, even if that rank is not >>> # part of the group. >>> # xdoctest: +SKIP("distributed") >>> process_groups = [torch.distributed.new_group(pids) for pids in [r1, r2]] >>> process_group = process_groups[0 if dist.get_rank() <= 3 else 1] >>> sync_bn_module = torch.nn.SyncBatchNorm.convert_sync_batchnorm(module, process_group) N)rqconfig)rr4nnmodules batchnormrr rrrrrrno_gradr-r.r/r0rhasattrrnamed_children add_moduleconvert_sync_batchnorm)clsmoduler module_outputnamechilds&&& r&r$SyncBatchNorm.convert_sync_batchnormysDH fehh..88CC D D!HH22##  **[[,3M}}}]]__+1==M()/M&%*0)<)rs *UU8! | | ~H H VL$OYL$^KT*KT\!TmZ!THLN*LN^!NmZ!NHLN*LN^!NmZ!NHmJmr)