# -*- coding: utf-8 -*-
import torch.nn as nn
import torch.nn.functional as F
#%% activation functions
[docs]def ReLU():
#here 'inplace=True' is used to save GPU memory
return nn.ReLU(inplace=True)
[docs]def LeakyReLU():
return nn.LeakyReLU(inplace=True)
[docs]def PReLU():
return nn.PReLU()
[docs]def RReLU():
return nn.RReLU(inplace=True)
[docs]def ReLU6():
return nn.ReLU6(inplace=True)
[docs]def ELU():
return nn.ELU(inplace=True)
[docs]class ELU_1(nn.Module):
def __init__(self):
super(ELU_1, self).__init__()
[docs] def forward(self, x):
x = F.elu(x)
x = x + 1
return x
[docs]def elu_1():
return ELU_1()
[docs]def CELU():
return nn.CELU(inplace=True)
[docs]def SELU():
return nn.SELU(inplace=True)
[docs]def SiLU():
return nn.SiLU(inplace=True)
[docs]def Sigmoid():
return nn.Sigmoid()
[docs]def LogSigmoid():
return nn.LogSigmoid()
[docs]def Tanh():
return nn.Tanh()
[docs]def Tanhshrink():
return nn.Tanhshrink()
[docs]def Softsign():
return nn.Softsign()
[docs]def Softplus():
return nn.Softplus()
[docs]class Softplus_1(nn.Module):
def __init__(self):
super(Softplus_1, self).__init__()
[docs] def forward(self, x):
x = F.softplus(x)
x = x - 1
return x
[docs]def softplus_1():
return Softplus_1()
[docs]class Softplus_2(nn.Module):
def __init__(self):
super(Softplus_2, self).__init__()
[docs] def forward(self, x):
x = F.softplus(x)
x = x - 2
return x
[docs]def softplus_2():
return Softplus_2()
[docs]class Sigmoid_1(nn.Module):
def __init__(self):
super(Sigmoid_1, self).__init__()
[docs] def forward(self, x):
x = F.sigmoid(x)
x = x - 0.5
return x
[docs]def sigmoid_1():
return Sigmoid_1()
[docs]def activation(activation_name='RReLU'):
"""Activation functions.
Parameters
----------
activation_name : str, optional
The name of activation function, which can be 'ReLU', 'LeakyReLU', 'PReLU',
'RReLU', 'ReLU6', 'ELU', 'CELU', 'SELU', 'SiLU', 'Sigmoid', 'LogSigmoid',
'Tanh', 'Tanhshrink', 'Softsign', or 'Softplus'. Default: 'RReLU'
Returns
-------
object
Activation function.
"""
return eval('%s()'%activation_name)
#%% Pooling
[docs]def maxPool1d(kernel_size):
return nn.MaxPool1d(kernel_size)
[docs]def maxPool2d(kernel_size):
return nn.MaxPool2d(kernel_size)
[docs]def maxPool3d(kernel_size):
return nn.MaxPool3d(kernel_size)
[docs]def avgPool1d(kernel_size):
return nn.AvgPool1d(kernel_size)
[docs]def avgPool2d(kernel_size):
return nn.AvgPool2d(kernel_size)
[docs]def avgPool3d(kernel_size):
return nn.AvgPool3d(kernel_size)
[docs]def pooling(pool_name='maxPool2d', kernel_size=2):
return eval('%s(kernel_size)'%pool_name)
#%% Dropout
[docs]def dropout():
return nn.Dropout(inplace=False)
[docs]def dropout2d():
return nn.Dropout2d(inplace=False)
[docs]def dropout3d():
return nn.Dropout3d(inplace=False)
[docs]def get_dropout(drouput_name='dropout'):
"""Get the dropout."""
return eval('%s()'%drouput_name)