【动手学深度学习-Pytorch版】长短期记忆网络LSTM

LSTM参数说明以及网络架构图

PS：时间仓促，有空补充内容~

LSTM从零开始实现

"""
遗忘门：相当于一个橡皮擦，决定保留昨天的哪些信息
输入门：相当于一个铅笔，再次根据昨天的记忆和今天的输入决定保留哪些信息
输出门：用于控制记忆细胞更新时所使用的输入信息
"""
import torch
from torch import nn, normal
from d2l import  torch as d2l
batch_size,num_steps = 32,35
train_iter,vocab = d2l.load_data_time_machine(batch_size,num_steps)
# 初始化模型参数
def get_lstm_params(vocab_size, num_hiddens, device):
    num_inputs = num_outputs = vocab_size

    def normal(shape):
        return torch.randn(size=shape, device=device)*0.01

    def three():
        return (normal((num_inputs, num_hiddens)),
                normal((num_hiddens, num_hiddens)),
                torch.zeros(num_hiddens, device=device))

    W_xi, W_hi, b_i = three()  # 输入门参数
    W_xf, W_hf, b_f = three()  # 遗忘门参数
    W_xo, W_ho, b_o = three()  # 输出门参数
    W_xc, W_hc, b_c = three()  # 候选记忆元参数
    # 输出层参数
    W_hq = normal((num_hiddens, num_outputs))
    b_q = torch.zeros(num_outputs, device=device)
    # 附加梯度
    params = [W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc,
              b_c, W_hq, b_q]
    for param in params:
        param.requires_grad_(True)
    return params
# 定义模型
"""在初始化函数中， 长短期记忆网络的隐状态需要返回一个额外的记忆元，
单元的值为0，形状为（批量大小，隐藏单元数）。 因此，我们得到以下的状态初始化。"""
def init_lstm_state(batch_size,num_hiddens,device):
    return (torch.zeros((batch_size,num_hiddens),device=device),
            torch.zeros((batch_size,num_hiddens),device=device))
def lstm(inputs,state,params):
    [W_xi,W_hi,b_i,W_xf,W_hf,b_f,W_xo,W_ho,b_o,W_xc,W_hc,b_c,W_hq,b_q] = params
    (H,C) = state
    outputs = []
    for X in inputs:
        I = torch.sigmoid((X@W_xi) + (H@W_hi) + b_i)
        F = torch.sigmoid((X@W_xf) + (H@W_hf) + b_f)
        O = torch.sigmoid((X@W_xo) + (H@W_ho) + b_o)
        C_tilda = torch.tanh((X@W_xc) + (H@W_hc) + b_c)
        C =  F * C + I * C_tilda
        H = O * torch.tanh(C)
        Y = (H @ W_hq) + b_q
        outputs.append(Y)
    return torch.cat(outputs,dim=0),(H,C)

# 训练和预测
vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()
num_epochs, lr = 500, 1
model = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_lstm_params,
                            init_lstm_state, lstm)
d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

LSTM的简洁实现

import torch
from torch import nn, normal
from d2l import  torch as d2l
batch_size,num_steps = 32,35
train_iter,vocab = d2l.load_data_time_machine(batch_size,num_steps)


# 训练和预测
vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()
num_epochs, lr = 500, 1

num_inputs = vocab_size
lstm_layer = nn.LSTM(num_inputs, num_hiddens)
model = d2l.RNNModel(lstm_layer, len(vocab))
model = model.to(device)

d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17