# 自动求导

import torch


## 简单情况的自动求导

x = Variable(torch.Tensor([2]), requires_grad=True)
y = x + 2
z = y ** 2 + 3
print(z)

Variable containing:
19
[torch.FloatTensor of size 1]


$$z = (x + 2)^2 + 3$$

$$\frac{\partial z}{\partial x} = 2 (x + 2) = 2 (2 + 2) = 8$$ 如果你对求导不熟悉，可以查看以下网址进行复习

# 使用自动求导
z.backward()

Variable containing:
8
[torch.FloatTensor of size 1]


x = Variable(torch.randn(10, 20), requires_grad=True)

out = torch.mean(y - torch.matmul(x, w)) # torch.matmul 是做矩阵乘法
out.backward()


# 得到 x 的梯度

Variable containing:

Columns 0 to 9
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172
-0.0600 -0.0242 -0.0514  0.0882  0.0056 -0.0400 -0.0300 -0.0052 -0.0289 -0.0172

Columns 10 to 19
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
-0.0372  0.0144 -0.1074 -0.0363 -0.0189  0.0209  0.0618  0.0435 -0.0591  0.0103
[torch.FloatTensor of size 10x20]

# 得到 y 的的梯度

Variable containing:
1.00000e-02 *
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
2.0000  2.0000  2.0000  2.0000  2.0000
[torch.FloatTensor of size 10x5]

# 得到 w 的梯度

Variable containing:
0.1342  0.1342  0.1342  0.1342  0.1342
0.0507  0.0507  0.0507  0.0507  0.0507
0.0328  0.0328  0.0328  0.0328  0.0328
-0.0086 -0.0086 -0.0086 -0.0086 -0.0086
0.0734  0.0734  0.0734  0.0734  0.0734
-0.0042 -0.0042 -0.0042 -0.0042 -0.0042
0.0078  0.0078  0.0078  0.0078  0.0078
-0.0769 -0.0769 -0.0769 -0.0769 -0.0769
0.0672  0.0672  0.0672  0.0672  0.0672
0.1614  0.1614  0.1614  0.1614  0.1614
-0.0042 -0.0042 -0.0042 -0.0042 -0.0042
-0.0970 -0.0970 -0.0970 -0.0970 -0.0970
-0.0364 -0.0364 -0.0364 -0.0364 -0.0364
-0.0419 -0.0419 -0.0419 -0.0419 -0.0419
0.0134  0.0134  0.0134  0.0134  0.0134
-0.0251 -0.0251 -0.0251 -0.0251 -0.0251
0.0586  0.0586  0.0586  0.0586  0.0586
-0.0050 -0.0050 -0.0050 -0.0050 -0.0050
0.1125  0.1125  0.1125  0.1125  0.1125
-0.0096 -0.0096 -0.0096 -0.0096 -0.0096
[torch.FloatTensor of size 20x5]


## 复杂情况的自动求导

m = Variable(torch.FloatTensor([[2, 3]]), requires_grad=True) # 构建一个 1 x 2 的矩阵
n = Variable(torch.zeros(1, 2)) # 构建一个相同大小的 0 矩阵
print(m)
print(n)

Variable containing:
2  3
[torch.FloatTensor of size 1x2]

Variable containing:
0  0
[torch.FloatTensor of size 1x2]

# 通过 m 中的值计算新的 n 中的值
n[0, 0] = m[0, 0] ** 2
n[0, 1] = m[0, 1] ** 3
print(n)

Variable containing:
4  27
[torch.FloatTensor of size 1x2]


$$n = (n_0,\ n_1) = (m_0^2,\ m_1^3) = (2^2,\ 3^3)$$

$$\frac{\partial n}{\partial m} = \frac{\partial (n_0,\ n_1)}{\partial (m_0,\ m_1)}$$

$$\frac{\partial n}{\partial m_0} = w_0 \frac{\partial n_0}{\partial m_0} + w_1 \frac{\partial n_1}{\partial m_0}$$

$$\frac{\partial n}{\partial m_1} = w_0 \frac{\partial n_0}{\partial m_1} + w_1 \frac{\partial n_1}{\partial m_1}$$

n.backward(torch.ones_like(n)) # 将 (w0, w1) 取成 (1, 1)

print(m.grad)

Variable containing:
4  27
[torch.FloatTensor of size 1x2]


$$\frac{\partial n}{\partial m_0} = w_0 \frac{\partial n_0}{\partial m_0} + w_1 \frac{\partial n_1}{\partial m_0} = 2 m_0 + 0 = 2 \times 2 = 4$$

$$\frac{\partial n}{\partial m_1} = w_0 \frac{\partial n_0}{\partial m_1} + w_1 \frac{\partial n_1}{\partial m_1} = 0 + 3 m_1^2 = 3 \times 3^2 = 27$$ 通过验算我们可以得到相同的结果

## 多次自动求导

x = Variable(torch.FloatTensor([3]), requires_grad=True)
y = x * 2 + x ** 2 + 3
print(y)

Variable containing:
18
[torch.FloatTensor of size 1]

y.backward(retain_graph=True) # 设置 retain_graph 为 True 来保留计算图

print(x.grad)

Variable containing:
8
[torch.FloatTensor of size 1]

y.backward() # 再做一次自动求导，这次不保留计算图

print(x.grad)

Variable containing:
16
[torch.FloatTensor of size 1]


$$x = \left[ \begin{matrix} x_0 \ x_1 \end{matrix} \right] = \left[ \begin{matrix} 2 \ 3 \end{matrix} \right]$$

$$k = (k_0,\ k_1) = (x_0^2 + 3 x_1,\ 2 x_0 + x_1^2)$$

$$j = \left[ \begin{matrix} \frac{\partial k_0}{\partial x_0} & \frac{\partial k_0}{\partial x_1} \ \frac{\partial k_1}{\partial x_0} & \frac{\partial k_1}{\partial x_1} \end{matrix} \right]$$

$$\left[ \begin{matrix} 4 & 3 \ 2 & 6 \ \end{matrix} \right]$$

x = Variable(torch.FloatTensor([2, 3]), requires_grad=True)
k = Variable(torch.zeros(2))

k[0] = x[0] ** 2 + 3 * x[1]
k[1] = x[1] ** 2 + 2 * x[0]

print(k)

Variable containing:
13
13
[torch.FloatTensor of size 2]

j = torch.zeros(2, 2)

k.backward(torch.FloatTensor([1, 0]), retain_graph=True)

k.backward(torch.FloatTensor([0, 1]))

print(j)

 4  3