Sigmoid() and Softmax() in PyTorch

#pytorch #sigmoid #softmax #activationfunction

*Memos:

My post explains Tanh, Softsign, Sigmoid and Softmax.
My post explains heaviside() and Identity().
My post explains ReLU() and LeakyReLU().
My post explains PReLU() and ELU().
My post explains SELU() and CELU().
My post explains GELU() and Mish().
My post explains SiLU() and Softplus().
My post explains Tanh() and Softsign().

Sigmoid() can get the 0D or more D tensor of the zero or more values computed by Sigmoid function from the 0D or more D tensor of zero or more elements as shown below:
*Memos:

The 1st argument is input(Required-Type:tensor of int, float, complex or bool). *A float tensor is returned except for a complex input tensor.
You can also use torch.sigmoid() which is the alias of torch.special.expit().

import torch
from torch import nn

my_tensor = torch.tensor([8., -3., 0., 1., 5., -2., -1., 4.])

sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([0.9997, 0.0474, 0.5000, 0.7311, 0.9933, 0.1192, 0.2689, 0.9820])

sigmoid
# Sigmoid()

my_tensor = torch.tensor([[8., -3., 0., 1.],
                          [5., -2., -1., 4.]])
sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([[0.9997, 0.0474, 0.5000, 0.7311],
#         [0.9933, 0.1192, 0.2689, 0.9820]])

my_tensor = torch.tensor([[[8., -3.], [0., 1.]],
                          [[5., -2.], [-1., 4.]]])
sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([[[0.9997, 0.0474], [0.5000, 0.7311]],
#         [[0.9933, 0.1192], [0.2689, 0.9820]]])

my_tensor = torch.tensor([[[8, -3], [0, 1]],
                          [[5, -2], [-1, 4]]])
sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([[[0.9997, 0.0474], [0.5000, 0.7311]],
#         [[0.9933, 0.1192], [0.2689, 0.9820]]])

my_tensor = torch.tensor([[[8.+0.j, -3.+0.j], [0.+0.j, 1.+0.j]],
                          [[5.+0.j, -2.+0.j], [-1.+0.j, 4.+0.j]]])
sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([[[0.9997-0.j, 0.0474-0.j], [0.5000-0.j, 0.7311-0.j]],
#         [[0.9933-0.j, 0.1192-0.j], [0.2689-0.j, 0.9820-0.j]]])

my_tensor = torch.tensor([[[True, False], [True, False]],
                          [[False, True], [False, True]]])
sigmoid = nn.Sigmoid()
sigmoid(input=my_tensor)
# tensor([[[0.7311, 0.5000], [0.7311, 0.5000]],
#         [[0.5000, 0.7311], [0.5000, 0.7311]]])

Softmax() can get the 0D or more D tensor of the zero or more values computed by Softmax function from the 0D or more D tensor of zero or more elements as shown below:
*Memos:

The 1st argument for initialization is dim(Required-Type:float). *Unsetting it works but there is a warning and the way is deprecated.
The 1st argument is input(Required-Type:tensor of float).
You can also use torch.nn.functional.softmax() with a tensor.

import torch
from torch import nn

my_tensor = torch.tensor([8., -3., 0., 1., 5., -2., -1., 4.])

softmax = nn.Softmax(dim=0)
softmax(input=my_tensor)
my_tensor.softmax(dim=0)
# tensor([9.3499e-01, 1.5616e-05, 3.1365e-04, 8.5260e-04,
#         4.6550e-02, 4.2448e-05,1.1539e-04, 1.7125e-02])

softmax
# Softmax(dim=0)

softmax.dim
# 0

softmax = nn.Softmax(dim=-1)
softmax(input=my_tensor)
# tensor([9.3499e-01, 1.5616e-05, 3.1365e-04, 8.5260e-04,
#         4.6550e-02, 4.2448e-05, 1.1539e-04, 1.7125e-02])

my_tensor = torch.tensor([[8., -3., 0., 1.],
                          [5., -2., -1., 4.]])
softmax = nn.Softmax(dim=0)
softmax(input=my_tensor)
# tensor([[0.9526, 0.2689, 0.7311, 0.0474],
#         [0.0474, 0.7311, 0.2689, 0.9526]])

softmax = nn.Softmax(dim=-2)
softmax(input=my_tensor)
# tensor([[0.9526, 0.2689, 0.7311, 0.0474],
#         [0.0474, 0.7311, 0.2689, 0.9526]])

softmax = nn.Softmax(dim=1)
softmax(input=my_tensor)
# tensor([[9.9874e-01, 1.6681e-05, 3.3504e-04, 9.1073e-04],
#         [7.2925e-01, 6.6499e-04, 1.8076e-03, 2.6828e-01]])

softmax = nn.Softmax(dim=-1)
softmax(input=my_tensor)
# tensor([[9.9874e-01, 1.6681e-05, 3.3504e-04, 9.1073e-04],
#         [7.2925e-01, 6.6499e-04, 1.8076e-03, 2.6828e-01]])

my_tensor = torch.tensor([[[8., -3.], [0., 1.]],
                          [[5., -2.], [-1., 4.]]])
softmax = nn.Softmax(dim=0)
softmax(input=my_tensor)
# tensor([[[0.9526, 0.2689], [0.7311, 0.0474]],
#         [[0.0474, 0.7311], [0.2689, 0.9526]]])

softmax = nn.Softmax(dim=-3)
softmax(input=my_tensor)
# tensor([[[0.9526, 0.2689], [0.7311, 0.0474]],
#         [[0.0474, 0.7311], [0.2689, 0.9526]]])

softmax = nn.Softmax(dim=1)
softmax(input=my_tensor)
# tensor([[[9.9966e-01, 1.7986e-02], [3.3535e-04, 9.8201e-01]],
#         [[9.9753e-01, 2.4726e-03], [2.4726e-03, 9.9753e-01]]])

softmax = nn.Softmax(dim=-2)
softmax(input=my_tensor)
# tensor([[[9.9966e-01, 1.7986e-02], [3.3535e-04, 9.8201e-01]],
#         [[9.9753e-01, 2.4726e-03], [2.4726e-03, 9.9753e-01]]])

softmax = nn.Softmax(dim=2)
softmax(input=my_tensor)
# tensor([[[9.9998e-01, 1.6701e-05], [2.6894e-01, 7.3106e-01]],
#         [[9.9909e-01, 9.1105e-04], [6.6929e-03, 9.9331e-01]]])

softmax = nn.Softmax(dim=-1)
softmax(input=my_tensor)
# tensor([[[9.9998e-01, 1.6701e-05], [2.6894e-01, 7.3106e-01]],
#         [[9.9909e-01, 9.1105e-04], [6.6929e-03, 9.9331e-01]]])

DEV Community

Sigmoid() and Softmax() in PyTorch

Top comments (0)

Read next

Enable Microsoft 365 Backups

Tokens vs Chunks

How to Actually Benefit from an Accelerator as a Digital Tech Startup

Rust🦀 takes decisions through control flow, so I learned it.