相關

關於 softmax + cross-entropy 的詳細介紹, 請參考 : BrightLamp. 多標籤softmax + cross-entropy交叉熵損失函式詳解及反向傳播中的梯度求導[EB/OL]. https://blog.csdn.net/oBrightLamp/article/details/84069835.

正文

變數有仔細命名, 註釋我就不寫了 ^ _ ^

import torch
import numpy as np


class SolfmaxEntropyLoss:
    def __init__(self):
        self.nx = None
        self.
 
ny = None
        self.softmax = None
        self.entropy = None
        self.loss = None
        self.dnx = None

    def __call__(self, nx, ny):
        self.nx = nx
        self.ny = ny
        shifted_x = nx - np.max(nx)
        ex = np.exp(shifted_x)
        sum_ex = np.sum(ex)
        self.softmax =
 
 ex / sum_ex
        self.entropy = - np.log(self.softmax) * ny
        self.loss = np.sum(self.entropy)
        return self.loss

    def backward(self):
        self.dnx = self.softmax.copy() * np.sum(self.ny)
        self.dnx -= self.ny
        return self.dnx


np.set_printoptions(precision=8, suppress=
 
True, linewidth=120)

x_numpy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float)
y_numpy = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], dtype=np.float)
x_tensor = torch.tensor(x_numpy, requires_grad=True)
y_tensor = torch.tensor(y_numpy)

solfmax_entropy_loss = SolfmaxEntropyLoss()
loss_numpy = solfmax_entropy_loss(x_numpy, y_numpy)
dx_numpy = solfmax_entropy_loss.backward()

log_softmax_layer = torch.nn.LogSoftmax(dim=0)
log_softmax_tensor = log_softmax_layer(x_tensor)
entropy_tensor = - log_softmax_tensor * y_tensor
loss_tensor = entropy_tensor.sum()
loss_tensor.backward()
dx_tensor = x_tensor.grad

print(loss_numpy)
print(loss_tensor.data.numpy())
print()
print(dx_numpy)
print(dx_tensor.data.numpy())

"""
程式碼輸出:
14.0634827759
14.063482775853203

[-0.09904564 -0.19740579 -0.29294821 -0.38083126 -0.44789396 -0.45836109 -0.31498552  0.24657787  1.9448936 ]
[-0.09904564 -0.19740579 -0.29294821 -0.38083126 -0.44789396 -0.45836109 -0.31498552  0.24657787  1.9448936 ]
"""