Shortcuts

Source code for torch.nn.modules.activation

import warnings
import torch
from torch.nn.parameter import Parameter

from .module import Module
from .. import functional as F
from ..._jit_internal import weak_module, weak_script_method


[docs]@torch._jit_internal.weak_module class Threshold(Module): __constants__ = ['threshold', 'value', 'inplace'] r"""Thresholds each element of the input Tensor Threshold is defined as: .. math:: y = \begin{cases} x, &\text{ if } x > \text{threshold} \\ \text{value}, &\text{ otherwise } \end{cases} Args: threshold: The value to threshold at value: The value to replace with inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Examples:: >>> m = nn.Threshold(0.1, 20) >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, threshold, value, inplace=False): super(Threshold, self).__init__() self.threshold = threshold self.value = value self.inplace = inplace # TODO: check in THNN (if inplace == True, then assert value <= threshold) @torch._jit_internal.weak_script_method def forward(self, input): return F.threshold(input, self.threshold, self.value, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'threshold={}, value={}{}'.format( self.threshold, self.value, inplace_str )
[docs]class ReLU(Threshold): r"""Applies the rectified linear unit function element-wise :math:`\text{ReLU}(x)= \max(0, x)` .. image:: scripts/activation_images/ReLU.png Args: inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Examples:: >>> m = nn.ReLU() >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, inplace=False): super(ReLU, self).__init__(0, 0, inplace) def extra_repr(self): inplace_str = 'inplace' if self.inplace else '' return inplace_str
[docs]@torch._jit_internal.weak_module class RReLU(Module): r"""Applies the randomized leaky rectified liner unit function, element-wise, as described in the paper: `Empirical Evaluation of Rectified Activations in Convolutional Network`_. The function is defined as: .. math:: \text{RReLU}(x) = \begin{cases} x & \text{if } x \geq 0 \\ ax & \text{ otherwise } \end{cases} where :math:`a` is randomly sampled from uniform distribution :math:`\mathcal{U}(\text{lower}, \text{upper})`. See: https://arxiv.org/pdf/1505.00853.pdf Args: lower: lower bound of the uniform distribution. Default: :math:`\frac{1}{8}` upper: upper bound of the uniform distribution. Default: :math:`\frac{1}{3}` inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Examples:: >>> m = nn.RReLU(0.1, 0.3) >>> input = torch.randn(2) >>> output = m(input) .. _`Empirical Evaluation of Rectified Activations in Convolutional Network`: https://arxiv.org/abs/1505.00853 """ __constants__ = ['lower', 'upper', 'inplace', 'training'] def __init__(self, lower=1. / 8, upper=1. / 3, inplace=False): super(RReLU, self).__init__() self.lower = lower self.upper = upper self.inplace = inplace @torch._jit_internal.weak_script_method def forward(self, input): return F.rrelu(input, self.lower, self.upper, self.training, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'lower={}, upper={}{}'.format(self.lower, self.upper, inplace_str)
[docs]class Hardtanh(Module): r"""Applies the HardTanh function element-wise HardTanh is defined as: .. math:: \text{HardTanh}(x) = \begin{cases} 1 & \text{ if } x > 1 \\ -1 & \text{ if } x < -1 \\ x & \text{ otherwise } \\ \end{cases} The range of the linear region :math:`[-1, 1]` can be adjusted using :attr:`min_val` and :attr:`max_val`. .. image:: scripts/activation_images/Hardtanh.png Args: min_val: minimum value of the linear region range. Default: -1 max_val: maximum value of the linear region range. Default: 1 inplace: can optionally do the operation in-place. Default: ``False`` Keyword arguments :attr:`min_value` and :attr:`max_value` have been deprecated in favor of :attr:`min_val` and :attr:`max_val`. Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Examples:: >>> m = nn.Hardtanh(-2, 2) >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, min_val=-1, max_val=1, inplace=False, min_value=None, max_value=None): super(Hardtanh, self).__init__() if min_value is not None: warnings.warn("keyword argument min_value is deprecated and renamed to min_val") min_val = min_value if max_value is not None: warnings.warn("keyword argument max_value is deprecated and renamed to max_val") max_val = max_value self.min_val = min_val self.max_val = max_val self.inplace = inplace assert self.max_val > self.min_val def forward(self, input): return F.hardtanh(input, self.min_val, self.max_val, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'min_val={}, max_val={}{}'.format( self.min_val, self.max_val, inplace_str )
[docs]class ReLU6(Hardtanh): r"""Applies the element-wise function: .. math:: \text{ReLU6}(x) = \min(\max(0,x), 6) Args: inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/ReLU6.png Examples:: >>> m = nn.ReLU6() >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, inplace=False): super(ReLU6, self).__init__(0, 6, inplace) def extra_repr(self): inplace_str = 'inplace' if self.inplace else '' return inplace_str
[docs]@torch._jit_internal.weak_module class Sigmoid(Module): r"""Applies the element-wise function: .. math:: \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Sigmoid.png Examples:: >>> m = nn.Sigmoid() >>> input = torch.randn(2) >>> output = m(input) """ @torch._jit_internal.weak_script_method def forward(self, input): return torch.sigmoid(input)
[docs]@torch._jit_internal.weak_module class Tanh(Module): r"""Applies the element-wise function: .. math:: \text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Tanh.png Examples:: >>> m = nn.Tanh() >>> input = torch.randn(2) >>> output = m(input) """ @torch._jit_internal.weak_script_method def forward(self, input): return torch.tanh(input)
[docs]class ELU(Module): r"""Applies the element-wise function: .. math:: \text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) Args: alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0 inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/ELU.png Examples:: >>> m = nn.ELU() >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, alpha=1., inplace=False): super(ELU, self).__init__() self.alpha = alpha self.inplace = inplace def forward(self, input): return F.elu(input, self.alpha, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'alpha={}{}'.format(self.alpha, inplace_str)
[docs]class CELU(Module): r"""Applies the element-wise function: .. math:: \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1)) More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ . Args: alpha: the :math:`\alpha` value for the CELU formulation. Default: 1.0 inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/CELU.png Examples:: >>> m = nn.CELU() >>> input = torch.randn(2) >>> output = m(input) .. _`Continuously Differentiable Exponential Linear Units`: https://arxiv.org/abs/1704.07483 """ def __init__(self, alpha=1., inplace=False): super(CELU, self).__init__() self.alpha = alpha self.inplace = inplace def forward(self, input): return F.celu(input, self.alpha, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'alpha={}{}'.format(self.alpha, inplace_str)
[docs]class SELU(Module): r"""Applied element-wise, as: .. math:: \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1))) with :math:`\alpha = 1.6732632423543772848170429916717` and :math:`\text{scale} = 1.0507009873554804934193349852946`. .. image:: scripts/activation_images/SELU.png More details can be found in the paper `Self-Normalizing Neural Networks`_ . Args: inplace (bool, optional): can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Examples:: >>> m = nn.SELU() >>> input = torch.randn(2) >>> output = m(input) .. _Self-Normalizing Neural Networks: https://arxiv.org/abs/1706.02515 """ def __init__(self, inplace=False): super(SELU, self).__init__() self.inplace = inplace def forward(self, input): return F.selu(input, self.inplace) def extra_repr(self): inplace_str = 'inplace' if self.inplace else '' return inplace_str
class GLU(Module): r"""Applies the gated linear unit function :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half of the input vector and :math:`b` is the second half. Args: dim (int): the dimension on which to split the input. Default: -1 Shape: - Input: :math:`(*, N, *)` where `*` means, any number of additional dimensions - Output: :math:`(*, N / 2, *)` Examples:: >>> m = nn.GLU() >>> input = torch.randn(4, 2) >>> output = m(input) """ def __init__(self, dim=-1): super(GLU, self).__init__() self.dim = dim def forward(self, input): return F.glu(input, self.dim) def extra_repr(self): return 'dim={}'.format(self.dim)
[docs]@torch._jit_internal.weak_module class Hardshrink(Module): r"""Applies the hard shrinkage function element-wise: .. math:: \text{HardShrink}(x) = \begin{cases} x, & \text{ if } x > \lambda \\ x, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} Args: lambd: the :math:`\lambda` value for the Hardshrink formulation. Default: 0.5 Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Hardshrink.png Examples:: >>> m = nn.Hardshrink() >>> input = torch.randn(2) >>> output = m(input) """ __constants__ = ['lambd'] def __init__(self, lambd=0.5): super(Hardshrink, self).__init__() self.lambd = lambd @torch._jit_internal.weak_script_method def forward(self, input): return F.hardshrink(input, self.lambd) def extra_repr(self): return '{}'.format(self.lambd)
[docs]class LeakyReLU(Module): r"""Applies the element-wise function: .. math:: \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x) or .. math:: \text{LeakyRELU}(x) = \begin{cases} x, & \text{ if } x \geq 0 \\ \text{negative\_slope} \times x, & \text{ otherwise } \end{cases} Args: negative_slope: Controls the angle of the negative slope. Default: 1e-2 inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/LeakyReLU.png Examples:: >>> m = nn.LeakyReLU(0.1) >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, negative_slope=1e-2, inplace=False): super(LeakyReLU, self).__init__() self.negative_slope = negative_slope self.inplace = inplace def forward(self, input): return F.leaky_relu(input, self.negative_slope, self.inplace) def extra_repr(self): inplace_str = ', inplace' if self.inplace else '' return 'negative_slope={}{}'.format(self.negative_slope, inplace_str)
[docs]class LogSigmoid(Module): r"""Applies the element-wise function: .. math:`\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/LogSigmoid.png Examples:: >>> m = nn.LogSigmoid() >>> input = torch.randn(2) >>> output = m(input) """ def forward(self, input): return F.logsigmoid(input)
[docs]class Softplus(Module): r"""Applies the element-wise function: .. math:: \text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x)) SoftPlus is a smooth approximation to the ReLU function and can be used to constrain the output of a machine to always be positive. For numerical stability the implementation reverts to the linear function for inputs above a certain value. Args: beta: the :math:`\beta` value for the Softplus formulation. Default: 1 threshold: values above this revert to a linear function. Default: 20 Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Softplus.png Examples:: >>> m = nn.Softplus() >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, beta=1, threshold=20): super(Softplus, self).__init__() self.beta = beta self.threshold = threshold def forward(self, input): return F.softplus(input, self.beta, self.threshold) def extra_repr(self): return 'beta={}, threshold={}'.format(self.beta, self.threshold)
[docs]@torch._jit_internal.weak_module class Softshrink(Module): r"""Applies the soft shrinkage function elementwise: .. math:: \text{SoftShrinkage}(x) = \begin{cases} x - \lambda, & \text{ if } x > \lambda \\ x + \lambda, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} Args: lambd: the :math:`\lambda` value for the Softshrink formulation. Default: 0.5 Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Softshrink.png Examples:: >>> m = nn.Softshrink() >>> input = torch.randn(2) >>> output = m(input) """ __constants__ = ['lambd'] def __init__(self, lambd=0.5): super(Softshrink, self).__init__() self.lambd = lambd @torch._jit_internal.weak_script_method def forward(self, input): return F.softshrink(input, self.lambd) def extra_repr(self): return str(self.lambd)
[docs]@torch._jit_internal.weak_module class PReLU(Module): r"""Applies the element-wise function: .. math:: \text{PReLU}(x) = \max(0,x) + a * \min(0,x) or .. math:: \text{PReLU}(x) = \begin{cases} x, & \text{ if } x \geq 0 \\ ax, & \text{ otherwise } \end{cases} Here :math:`a` is a learnable parameter. When called without arguments, `nn.PReLU()` uses a single parameter :math:`a` across all input channels. If called with `nn.PReLU(nChannels)`, a separate :math:`a` is used for each input channel. .. note:: weight decay should not be used when learning :math:`a` for good performance. .. note:: Channel dim is the 2nd dim of input. When input has dims < 2, then there is no channel dim and the number of channels = 1. Args: num_parameters (int): number of :math:`a` to learn. Although it takes an int as input, there is only two values are legitimate: 1, or the number of channels at input. Default: 1 init (float): the initial value of :math:`a`. Default: 0.25 Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input Attributes: weight (Tensor): the learnable weights of shape (attr:`num_parameters`). The attr:`dtype` is default to .. image:: scripts/activation_images/PReLU.png Examples:: >>> m = nn.PReLU() >>> input = torch.randn(2) >>> output = m(input) """ def __init__(self, num_parameters=1, init=0.25): self.num_parameters = num_parameters super(PReLU, self).__init__() self.weight = Parameter(torch.Tensor(num_parameters).fill_(init)) @torch._jit_internal.weak_script_method def forward(self, input): return F.prelu(input, self.weight) def extra_repr(self): return 'num_parameters={}'.format(self.num_parameters)
[docs]@torch._jit_internal.weak_module class Softsign(Module): r"""Applies the element-wise function: .. math:: \text{SoftSign}(x) = \frac{x}{ 1 + |x|} Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Softsign.png Examples:: >>> m = nn.Softsign() >>> input = torch.randn(2) >>> output = m(input) """ @torch._jit_internal.weak_script_method def forward(self, input): return F.softsign(input)
[docs]@torch._jit_internal.weak_module class Tanhshrink(Module): r"""Applies the element-wise function: .. math:: \text{Tanhshrink}(x) = x - \text{Tanh}(x) Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: scripts/activation_images/Tanhshrink.png Examples:: >>> m = nn.Tanhshrink() >>> input = torch.randn(2) >>> output = m(input) """ @torch._jit_internal.weak_script_method def forward(self, input): return F.tanhshrink(input)
[docs]class Softmin(Module): r"""Applies the Softmin function to an n-dimensional input Tensor rescaling them so that the elements of the n-dimensional output Tensor lie in the range `(0, 1)` and sum to 1 .. math:: \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} Shape: - Input: any shape - Output: same as input Arguments: dim (int): A dimension along which Softmin will be computed (so every slice along dim will sum to 1). Returns: a Tensor of the same dimension and shape as the input, with values in the range [0, 1] Examples:: >>> m = nn.Softmin() >>> input = torch.randn(2, 3) >>> output = m(input) """ def __init__(self, dim=None): super(Softmin, self).__init__() self.dim = dim def forward(self, input): return F.softmin(input, self.dim, _stacklevel=5)
[docs]class Softmax(Module): r"""Applies the Softmax function to an n-dimensional input Tensor rescaling them so that the elements of the n-dimensional output Tensor lie in the range (0,1) and sum to 1 Softmax is defined as: .. math:: \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} Shape: - Input: any shape - Output: same as input Returns: a Tensor of the same dimension and shape as the input with values in the range [0, 1] Arguments: dim (int): A dimension along which Softmax will be computed (so every slice along dim will sum to 1). .. note:: This module doesn't work directly with NLLLoss, which expects the Log to be computed between the Softmax and itself. Use `LogSoftmax` instead (it's faster and has better numerical properties). Examples:: >>> m = nn.Softmax() >>> input = torch.randn(2, 3) >>> output = m(input) """ def __init__(self, dim=None): super(Softmax, self).__init__() self.dim = dim def __setstate__(self, state): self.__dict__.update(state) if not hasattr(self, 'dim'): self.dim = None def forward(self, input): return F.softmax(input, self.dim, _stacklevel=5)
[docs]class Softmax2d(Module): r"""Applies SoftMax over features to each spatial location. When given an image of ``Channels x Height x Width``, it will apply `Softmax` to each location :math:`(Channels, h_i, w_j)` Shape: - Input: :math:`(N, C, H, W)` - Output: :math:`(N, C, H, W)` (same shape as input) Returns: a Tensor of the same dimension and shape as the input with values in the range [0, 1] Examples:: >>> m = nn.Softmax2d() >>> # you softmax over the 2nd dimension >>> input = torch.randn(2, 3, 12, 13) >>> output = m(input) """ def forward(self, input): assert input.dim() == 4, 'Softmax2d requires a 4D tensor as input' return F.softmax(input, 1, _stacklevel=5)
[docs]class LogSoftmax(Module): r"""Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional input Tensor. The LogSoftmax formulation can be simplified as: .. math:: \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right) Shape: - Input: any shape - Output: same as input Arguments: dim (int): A dimension along which Softmax will be computed (so every slice along dim will sum to 1). Returns: a Tensor of the same dimension and shape as the input with values in the range [-inf, 0) Examples:: >>> m = nn.LogSoftmax() >>> input = torch.randn(2, 3) >>> output = m(input) """ def __init__(self, dim=None): super(LogSoftmax, self).__init__() self.dim = dim def __setstate__(self, state): self.__dict__.update(state) if not hasattr(self, 'dim'): self.dim = None def forward(self, input): return F.log_softmax(input, self.dim, _stacklevel=5)

Docs

Access comprehensive developer documentation for PyTorch

View Docs

Tutorials

Get in-depth tutorials for beginners and advanced developers

View Tutorials

Resources

Find development resources and get your questions answered

View Resources