# mypy: allow-untyped-defsimportwarningsimporttorch.nn.functionalasFfromtorchimportTensorfrom.batchnormimport_LazyNormBase,_NormBase__all__=["InstanceNorm1d","InstanceNorm2d","InstanceNorm3d","LazyInstanceNorm1d","LazyInstanceNorm2d","LazyInstanceNorm3d",]class_InstanceNorm(_NormBase):def__init__(self,num_features:int,eps:float=1e-5,momentum:float=0.1,affine:bool=False,track_running_stats:bool=False,device=None,dtype=None,)->None:factory_kwargs={"device":device,"dtype":dtype}super().__init__(num_features,eps,momentum,affine,track_running_stats,**factory_kwargs)def_check_input_dim(self,input):raiseNotImplementedErrordef_get_no_batch_dim(self):raiseNotImplementedErrordef_handle_no_batch_input(self,input):returnself._apply_instance_norm(input.unsqueeze(0)).squeeze(0)def_apply_instance_norm(self,input):returnF.instance_norm(input,self.running_mean,self.running_var,self.weight,self.bias,self.trainingornotself.track_running_stats,self.momentumifself.momentumisnotNoneelse0.0,self.eps,)def_load_from_state_dict(self,state_dict,prefix,local_metadata,strict,missing_keys,unexpected_keys,error_msgs,):version=local_metadata.get("version",None)# at version 1: removed running_mean and running_var when# track_running_stats=False (default)ifversionisNoneandnotself.track_running_stats:running_stats_keys=[]fornamein("running_mean","running_var"):key=prefix+nameifkeyinstate_dict:running_stats_keys.append(key)iflen(running_stats_keys)>0:error_msgs.append("Unexpected running stats buffer(s) {names} for {klass} ""with track_running_stats=False. If state_dict is a ""checkpoint saved before 0.4.0, this may be expected ""because {klass} does not track running stats by default ""since 0.4.0. Please remove these keys from state_dict. If ""the running stats are actually needed, instead set ""track_running_stats=True in {klass} to enable them. See ""the documentation of {klass} for details.".format(names=" and ".join(f'"{k}"'forkinrunning_stats_keys),klass=self.__class__.__name__,))forkeyinrunning_stats_keys:state_dict.pop(key)super()._load_from_state_dict(state_dict,prefix,local_metadata,strict,missing_keys,unexpected_keys,error_msgs,)defforward(self,input:Tensor)->Tensor:self._check_input_dim(input)feature_dim=input.dim()-self._get_no_batch_dim()ifinput.size(feature_dim)!=self.num_features:ifself.affine:raiseValueError(f"expected input's size at dim={feature_dim} to match num_features"f" ({self.num_features}), but got: {input.size(feature_dim)}.")else:warnings.warn(f"input's size at dim={feature_dim} does not match num_features. ""You can silence this warning by not passing in num_features, ""which is not used because affine=False")ifinput.dim()==self._get_no_batch_dim():returnself._handle_no_batch_input(input)returnself._apply_instance_norm(input)
[docs]classInstanceNorm1d(_InstanceNorm):r"""Applies Instance Normalization. This operation applies Instance Normalization over a 2D (unbatched) or 3D (batched) input as described in the paper `Instance Normalization: The Missing Ingredient for Fast Stylization <https://arxiv.org/abs/1607.08022>`__. .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension separately for each object in a mini-batch. :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the number of features or channels of the input) if :attr:`affine` is ``True``. The variance is calculated via the biased estimator, equivalent to `torch.var(input, unbiased=False)`. By default, this layer uses instance statistics computed from input data in both training and evaluation modes. If :attr:`track_running_stats` is set to ``True``, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. .. note:: :class:`InstanceNorm1d` and :class:`LayerNorm` are very similar, but have some subtle differences. :class:`InstanceNorm1d` is applied on each channel of channeled data like multidimensional time series, but :class:`LayerNorm` is usually applied on entire sample and often in NLP tasks. Additionally, :class:`LayerNorm` applies elementwise affine transform, while :class:`InstanceNorm1d` usually don't apply affine transform. Args: num_features: number of features or channels :math:`C` of the input eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, L)` or :math:`(C, L)` - Output: :math:`(N, C, L)` or :math:`(C, L)` (same shape as input) Examples:: >>> # Without Learnable Parameters >>> m = nn.InstanceNorm1d(100) >>> # With Learnable Parameters >>> m = nn.InstanceNorm1d(100, affine=True) >>> input = torch.randn(20, 100, 40) >>> output = m(input) """def_get_no_batch_dim(self):return2def_check_input_dim(self,input):ifinput.dim()notin(2,3):raiseValueError(f"expected 2D or 3D input (got {input.dim()}D input)")
[docs]classLazyInstanceNorm1d(_LazyNormBase,_InstanceNorm):r"""A :class:`torch.nn.InstanceNorm1d` module with lazy initialization of the ``num_features`` argument. The ``num_features`` argument of the :class:`InstanceNorm1d` is inferred from the ``input.size(1)``. The attributes that will be lazily initialized are `weight`, `bias`, `running_mean` and `running_var`. Check the :class:`torch.nn.modules.lazy.LazyModuleMixin` for further documentation on lazy modules and their limitations. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, L)` or :math:`(C, L)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, L)` or :math:`(C, L)` - Output: :math:`(N, C, L)` or :math:`(C, L)` (same shape as input) """cls_to_become=InstanceNorm1d# type: ignore[assignment]def_get_no_batch_dim(self):return2def_check_input_dim(self,input):ifinput.dim()notin(2,3):raiseValueError(f"expected 2D or 3D input (got {input.dim()}D input)")
[docs]classInstanceNorm2d(_InstanceNorm):r"""Applies Instance Normalization. This operation applies Instance Normalization over a 4D input (a mini-batch of 2D inputs with additional channel dimension) as described in the paper `Instance Normalization: The Missing Ingredient for Fast Stylization <https://arxiv.org/abs/1607.08022>`__. .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension separately for each object in a mini-batch. :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size `C` (where `C` is the input size) if :attr:`affine` is ``True``. The standard-deviation is calculated via the biased estimator, equivalent to `torch.var(input, unbiased=False)`. By default, this layer uses instance statistics computed from input data in both training and evaluation modes. If :attr:`track_running_stats` is set to ``True``, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. .. note:: :class:`InstanceNorm2d` and :class:`LayerNorm` are very similar, but have some subtle differences. :class:`InstanceNorm2d` is applied on each channel of channeled data like RGB images, but :class:`LayerNorm` is usually applied on entire sample and often in NLP tasks. Additionally, :class:`LayerNorm` applies elementwise affine transform, while :class:`InstanceNorm2d` usually don't apply affine transform. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, H, W)` or :math:`(C, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, H, W)` or :math:`(C, H, W)` - Output: :math:`(N, C, H, W)` or :math:`(C, H, W)` (same shape as input) Examples:: >>> # Without Learnable Parameters >>> m = nn.InstanceNorm2d(100) >>> # With Learnable Parameters >>> m = nn.InstanceNorm2d(100, affine=True) >>> input = torch.randn(20, 100, 35, 45) >>> output = m(input) """def_get_no_batch_dim(self):return3def_check_input_dim(self,input):ifinput.dim()notin(3,4):raiseValueError(f"expected 3D or 4D input (got {input.dim()}D input)")
[docs]classLazyInstanceNorm2d(_LazyNormBase,_InstanceNorm):r"""A :class:`torch.nn.InstanceNorm2d` module with lazy initialization of the ``num_features`` argument. The ``num_features`` argument of the :class:`InstanceNorm2d` is inferred from the ``input.size(1)``. The attributes that will be lazily initialized are `weight`, `bias`, `running_mean` and `running_var`. Check the :class:`torch.nn.modules.lazy.LazyModuleMixin` for further documentation on lazy modules and their limitations. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, H, W)` or :math:`(C, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, H, W)` or :math:`(C, H, W)` - Output: :math:`(N, C, H, W)` or :math:`(C, H, W)` (same shape as input) """cls_to_become=InstanceNorm2d# type: ignore[assignment]def_get_no_batch_dim(self):return3def_check_input_dim(self,input):ifinput.dim()notin(3,4):raiseValueError(f"expected 3D or 4D input (got {input.dim()}D input)")
[docs]classInstanceNorm3d(_InstanceNorm):r"""Applies Instance Normalization. This operation applies Instance Normalization over a 5D input (a mini-batch of 3D inputs with additional channel dimension) as described in the paper `Instance Normalization: The Missing Ingredient for Fast Stylization <https://arxiv.org/abs/1607.08022>`__. .. math:: y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The mean and standard-deviation are calculated per-dimension separately for each object in a mini-batch. :math:`\gamma` and :math:`\beta` are learnable parameter vectors of size C (where C is the input size) if :attr:`affine` is ``True``. The standard-deviation is calculated via the biased estimator, equivalent to `torch.var(input, unbiased=False)`. By default, this layer uses instance statistics computed from input data in both training and evaluation modes. If :attr:`track_running_stats` is set to ``True``, during training this layer keeps running estimates of its computed mean and variance, which are then used for normalization during evaluation. The running estimates are kept with a default :attr:`momentum` of 0.1. .. note:: This :attr:`momentum` argument is different from one used in optimizer classes and the conventional notion of momentum. Mathematically, the update rule for running statistics here is :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t`, where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value. .. note:: :class:`InstanceNorm3d` and :class:`LayerNorm` are very similar, but have some subtle differences. :class:`InstanceNorm3d` is applied on each channel of channeled data like 3D models with RGB color, but :class:`LayerNorm` is usually applied on entire sample and often in NLP tasks. Additionally, :class:`LayerNorm` applies elementwise affine transform, while :class:`InstanceNorm3d` usually don't apply affine transform. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` - Output: :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` (same shape as input) Examples:: >>> # Without Learnable Parameters >>> m = nn.InstanceNorm3d(100) >>> # With Learnable Parameters >>> m = nn.InstanceNorm3d(100, affine=True) >>> input = torch.randn(20, 100, 35, 45, 10) >>> output = m(input) """def_get_no_batch_dim(self):return4def_check_input_dim(self,input):ifinput.dim()notin(4,5):raiseValueError(f"expected 4D or 5D input (got {input.dim()}D input)")
[docs]classLazyInstanceNorm3d(_LazyNormBase,_InstanceNorm):r"""A :class:`torch.nn.InstanceNorm3d` module with lazy initialization of the ``num_features`` argument. The ``num_features`` argument of the :class:`InstanceNorm3d` is inferred from the ``input.size(1)``. The attributes that will be lazily initialized are `weight`, `bias`, `running_mean` and `running_var`. Check the :class:`torch.nn.modules.lazy.LazyModuleMixin` for further documentation on lazy modules and their limitations. Args: num_features: :math:`C` from an expected input of size :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` eps: a value added to the denominator for numerical stability. Default: 1e-5 momentum: the value used for the running_mean and running_var computation. Default: 0.1 affine: a boolean value that when set to ``True``, this module has learnable affine parameters, initialized the same way as done for batch normalization. Default: ``False``. track_running_stats: a boolean value that when set to ``True``, this module tracks the running mean and variance, and when set to ``False``, this module does not track such statistics and always uses batch statistics in both training and eval modes. Default: ``False`` Shape: - Input: :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` - Output: :math:`(N, C, D, H, W)` or :math:`(C, D, H, W)` (same shape as input) """cls_to_become=InstanceNorm3d# type: ignore[assignment]def_get_no_batch_dim(self):return4def_check_input_dim(self,input):ifinput.dim()notin(4,5):raiseValueError(f"expected 4D or 5D input (got {input.dim()}D input)")
Docs
Access comprehensive developer documentation for PyTorch
To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.