• 3、自然语言和单词的分布式表示（下）

            本节主要以实验为主

    一、学习数据的准备

    1、将语料库的文本转化为单词ID

    1. import numpy as np
    2.  
    3. def preprocess( text ):
    4.     text = text.lower()  # 大写字母变为小写字母
    5.     text = text.replace('.', ' .')  # 用‘ .’代替‘.’
    6.     words = text.split(' ')  # 以空格为分隔条件，对语句进行分隔。
    7.  
    8.     word_to_id = {}
    9.     id_to_word = {}
    10.     for word in words:
    11.         if word not in word_to_id:
    12.             new_id = len(word_to_id)
    13.             word_to_id[word] = new_id
    14.             id_to_word[new_id] = word
    15.     print(word_to_id)
    16.     corpus = np.array([word_to_id[w] for w in words])
    17.     print(corpus)
    18.  
    19. text = "You say goodbye and I say hello."
    20. preprocess(text)

            输出结果：

             word_to_id字典中保存的是单词与ID的对应关系；corpus数组中保存的是：将单词用ID替换。

    2、从单词ID列表corpus生成contexts和target

            word2vec中使用的神经网络的输入是上下文，它的正确解标签是被这些上下文包围在中间的单词，即目标词。我们要做的事情是：当向神经网络输入上下文时，使目标词出现的概率高。故从语料库生成上下文和目标词。

            代码实现：

    1. import numpy as np
    2.  
    3. def preprocess( text ):
    4.     text = text.lower()  # 大写字母变为小写字母
    5.     text = text.replace('.', ' .')  # 用‘ .’代替‘.’
    6.     words = text.split(' ')  # 以空格为分隔条件，对语句进行分隔。
    7.  
    8.     word_to_id = {}
    9.     id_to_word = {}
    10.     for word in words:
    11.         if word not in word_to_id:
    12.             new_id = len(word_to_id)
    13.             word_to_id[word] = new_id
    14.             id_to_word[new_id] = word
    15.  
    16.     corpus = np.array([word_to_id[w] for w in words])
    17.     return corpus, word_to_id, id_to_word
    18.  
    19. def create_conext_target(corpus, windows_size=1):  # 单词ID列表和上下文的窗口大小
    20.     target = corpus[windows_size:-windows_size]
    21.     contexts = []
    22.  
    23.     for idx in range(windows_size, len(corpus)-windows_size):
    24.         cs = []
    25.         for t in range(-windows_size, windows_size+1):
    26.             if t == 0:
    27.                 continue
    28.             cs.append(corpus[idx + t])
    29.         contexts.append(cs)
    30.  
    31.     return np.array(contexts), np.array(target)
    32.  
    33.  
    34.  
    35. text = "You say goodbye and I say hello."
    36. corpus, word_to_id, id_to_word = preprocess(text)
    37. contexts, target = create_conext_target(corpus, windows_size=1)
    38. print(contexts)
    39. print(target)

            结果：

    3、转化为one-hot表示

            将上下文和目标词转化为one-hot表示，如下图所示：

             代码实现：

    1. import numpy as np
    2.  
    3. def preprocess( text ): # 预处理：生成单词ID，单词ID列表
    4.     text = text.lower()  # 大写字母变为小写字母
    5.     text = text.replace('.', ' .')  # 用‘ .’代替‘.’
    6.     words = text.split(' ')  # 以空格为分隔条件，对语句进行分隔。
    7.  
    8.     word_to_id = {}
    9.     id_to_word = {}
    10.     for word in words:
    11.         if word not in word_to_id:
    12.             new_id = len(word_to_id)
    13.             word_to_id[word] = new_id
    14.             id_to_word[new_id] = word
    15.  
    16.     corpus = np.array([word_to_id[w] for w in words])
    17.     return corpus, word_to_id, id_to_word
    18.  
    19. def create_conext_target(corpus, windows_size=1):  # 生成上下文和目标词
    20.     target = corpus[windows_size:-windows_size]
    21.     contexts = []
    22.  
    23.     for idx in range(windows_size, len(corpus)-windows_size):
    24.         cs = []
    25.         for t in range(-windows_size, windows_size+1):
    26.             if t == 0:
    27.                 continue
    28.             cs.append(corpus[idx + t])
    29.         contexts.append(cs)
    30.  
    31.     return np.array(contexts), np.array(target)
    32.  
    33. def convert_one_hot(corpus, vocab_size): # 转化为one_hot表示
    34.     N = corpus.shape[0]  # shape返回各个维度上的元素个数
    35.     if corpus.ndim == 1:  # ndim返回数组的维度
    36.         one_hot = np.zeros((N, vocab_size), dtype=np.int32)  #生成一个N行vocab_size列的零矩阵
    37.         for idx, word_id in enumerate(corpus):
    38.             one_hot[idx, word_id] = 1
    39.  
    40.     elif corpus.ndim == 2:
    41.         C = corpus.shape[1]
    42.         one_hot = np.zeros((N, C, vocab_size), dtype=np.int32)
    43.         for idx_0, word_ids in enumerate(corpus):
    44.             for idx_1, word_id in enumerate(word_ids):
    45.                 one_hot[idx_0, idx_1, word_id] = 1
    46.  
    47.     return one_hot
    48.  
    49.  
    50. text = "You say goodbye and I say hello."
    51. corpus, word_to_id, id_to_word = preprocess(text)
    52. contexts, target = create_conext_target(corpus, windows_size=1)
    53.  
    54.  
    55. vocab_size = len(word_to_id)
    56. target = convert_one_hot(target, vocab_size)
    57. contexts = convert_one_hot(contexts, vocab_size)
    58. print(target)
    59. print(contexts)

            结果：

    二、CBOW模型的实现

            神经网络示意图：

     

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. import time
    4.  
    5. def preprocess( text ): # 预处理：生成单词ID，单词ID列表
    6.     text = text.lower()  # 大写字母变为小写字母
    7.     text = text.replace('.', ' .')  # 用‘ .’代替‘.’
    8.     words = text.split(' ')  # 以空格为分隔条件，对语句进行分隔。
    9.  
    10.     word_to_id = {}
    11.     id_to_word = {}
    12.     for word in words:
    13.         if word not in word_to_id:
    14.             new_id = len(word_to_id)
    15.             word_to_id[word] = new_id
    16.             id_to_word[new_id] = word
    17.  
    18.     corpus = np.array([word_to_id[w] for w in words])
    19.     return corpus, word_to_id, id_to_word
    20.  
    21. def create_conext_target(corpus, windows_size):  # 生成上下文和目标词
    22.     target = corpus[windows_size:-windows_size]
    23.     contexts = []
    24.  
    25.     for idx in range(windows_size, len(corpus)-windows_size):
    26.         cs = []
    27.         for t in range(-windows_size, windows_size+1):
    28.             if t == 0:
    29.                 continue
    30.             cs.append(corpus[idx + t])
    31.         contexts.append(cs)
    32.  
    33.     return np.array(contexts), np.array(target)
    34.  
    35. def convert_one_hot(corpus, vocab_size): # 转化为one_hot表示
    36.     N = corpus.shape[0]  # shape返回各个维度上的元素个数
    37.     if corpus.ndim == 1:  # ndim返回数组的维度
    38.         one_hot = np.zeros((N, vocab_size), dtype=np.int32)  #生成一个N行vocab_size列的零矩阵
    39.         for idx, word_id in enumerate(corpus):
    40.             one_hot[idx, word_id] = 1
    41.  
    42.     elif corpus.ndim == 2:
    43.         C = corpus.shape[1]
    44.         one_hot = np.zeros((N, C, vocab_size), dtype=np.int32)
    45.         for idx_0, word_ids in enumerate(corpus):
    46.             for idx_1, word_id in enumerate(word_ids):
    47.                 one_hot[idx_0, idx_1, word_id] = 1
    48.  
    49.     return one_hot
    50.  
    51. class MatMul:
    52.     def __init__(self, W):
    53.         self.params = [W]
    54.         self.grads = [np.zeros_like(W)]
    55.         self.x = None
    56.  
    57.     def forward(self, x):
    58.         W, = self.params
    59.         out = np.dot(x, W)
    60.         self.x = x
    61.         return out
    62.  
    63.     def backward(self, dout):
    64.         W, = self.params
    65.         dx = np.dot(dout, W.T)
    66.         dW = np.dot(self.x.T, dout)
    67.         self.grads[0][...] = dW
    68.         return dx
    69.  
    70. def softmax(x):
    71.     if x.ndim == 2:
    72.         x = x - x.max(axis=1, keepdims=True)
    73.         x = np.exp(x)
    74.         x /= x.sum(axis=1, keepdims=True)
    75.     elif x.ndim == 1:
    76.         x = x - np.max(x)
    77.         x = np.exp(x) / np.sum(np.exp(x))
    78.  
    79.     return x
    80.  
    81.  
    82. def cross_entropy_error(y, t):
    83.     if y.ndim == 1:
    84.         t = t.reshape(1, t.size)
    85.         y = y.reshape(1, y.size)
    86.  
    87.     # 在监督标签为one-hot-vector的情况下，转换为正确解标签的索引
    88.     if t.size == y.size:
    89.         t = t.argmax(axis=1)
    90.  
    91.     batch_size = y.shape[0]
    92.  
    93.     return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size
    94.  
    95.  
    96. class SoftmaxWithLoss:
    97.     def __init__(self):
    98.         self.params, self.grads = [], []
    99.         self.y = None  # softmax的输出
    100.         self.t = None  # 监督标签
    101.  
    102.     def forward(self, x, t):
    103.         self.t = t
    104.         self.y = softmax(x)
    105.  
    106.         # 在监督标签为one-hot向量的情况下，转换为正确解标签的索引
    107.         if self.t.size == self.y.size:
    108.             self.t = self.t.argmax(axis=1)
    109.  
    110.         loss = cross_entropy_error(self.y, self.t)
    111.         return loss
    112.  
    113.     def backward(self, dout):
    114.         batch_size = self.t.shape[0]
    115.  
    116.         dx = self.y.copy()
    117.         dx[np.arange(batch_size), self.t] -= 1
    118.         dx *= dout
    119.         dx = dx / batch_size
    120.         return dx
    121.  
    122. class Trainer:
    123.     def __init__(self, model, optimizer):
    124.         self.model = model
    125.         self.optimizer = optimizer
    126.         self.loss_list = []
    127.         self.eval_interval = None
    128.         self.current_epoch = 0
    129.  
    130.     def fit(self, x, t, max_epoch, batch_size, max_grad=None, eval_interval=20):
    131.         data_size = len(x)
    132.         print(data_size)
    133.         max_iters = data_size // batch_size
    134.         print(max_iters)
    135.         self.eval_interval = eval_interval
    136.         model, optimizer = self.model, self.optimizer
    137.         total_loss = 0
    138.         loss_count = 0
    139.  
    140.         start_time = time.time()
    141.         for epoch in range(max_epoch):
    142.             # 打乱
    143.             idx = np.random.permutation(np.arange(data_size))
    144.             x = x[idx]
    145.             t = t[idx]
    146.  
    147.             for iters in range(max_iters):
    148.                 batch_x = x[iters*batch_size:(iters+1)*batch_size]
    149.                 batch_t = t[iters*batch_size:(iters+1)*batch_size]
    150.  
    151.                 # 计算梯度，更新参数
    152.                 loss = model.forward(batch_x, batch_t)
    153.                 model.backward()
    154.                 params, grads = remove_duplicate(model.params, model.grads)  # 将共享的权重整合为1个
    155.                 if max_grad is not None:
    156.                     clip_grads(grads, max_grad)
    157.                 optimizer.update(params, grads)
    158.                 total_loss += loss
    159.                 loss_count += 1
    160.  
    161.                 # 评价
    162.                 if (eval_interval is not None) and (iters % eval_interval) == 0:
    163.                     avg_loss = total_loss / loss_count
    164.                     elapsed_time = time.time() - start_time
    165.                     print('| epoch %d |  iter %d / %d | time %d[s] | loss %.2f'
    166.                           % (self.current_epoch + 1, iters + 1, max_iters, elapsed_time, avg_loss))
    167.                     self.loss_list.append(float(avg_loss))
    168.                     total_loss, loss_count = 0, 0
    169.  
    170.             self.current_epoch += 1
    171.  
    172.     def plot(self, ylim=None):
    173.         x = np.arange(len(self.loss_list))
    174.         if ylim is not None:
    175.             plt.ylim(*ylim)
    176.         plt.plot(x, self.loss_list, label='train')
    177.         plt.xlabel('iterations (x' + str(self.eval_interval) + ')')
    178.         plt.ylabel('loss')
    179.         plt.show()
    180.  
    181. def remove_duplicate(params, grads):
    182.     '''
    183.     将参数列表中重复的权重整合为1个，
    184.     加上与该权重对应的梯度
    185.     '''
    186.     params, grads = params[:], grads[:]  # copy list
    187.  
    188.     while True:
    189.         find_flg = False
    190.         L = len(params)
    191.  
    192.         for i in range(0, L - 1):
    193.             for j in range(i + 1, L):
    194.                 # 在共享权重的情况下
    195.                 if params[i] is params[j]:
    196.                     grads[i] += grads[j]  # 加上梯度
    197.                     find_flg = True
    198.                     params.pop(j)
    199.                     grads.pop(j)
    200.                 # 在作为转置矩阵共享权重的情况下（weight tying）
    201.                 elif params[i].ndim == 2 and params[j].ndim == 2 and \
    202.                      params[i].T.shape == params[j].shape and np.all(params[i].T == params[j]):
    203.                     grads[i] += grads[j].T
    204.                     find_flg = True
    205.                     params.pop(j)
    206.                     grads.pop(j)
    207.  
    208.                 if find_flg: break
    209.             if find_flg: break
    210.  
    211.         if not find_flg: break
    212.  
    213.     return params, grads
    214.  
    215.  
    216. def clip_grads(grads, max_norm):
    217.     total_norm = 0
    218.     for grad in grads:
    219.         total_norm += np.sum(grad ** 2)
    220.     total_norm = np.sqrt(total_norm)
    221.  
    222.     rate = max_norm / (total_norm + 1e-6)
    223.     if rate < 1:
    224.         for grad in grads:
    225.             grad *= rate
    226.  
    227.  
    228. class SimpleCBOW:
    229.     def __init__(self, vocab_size, hidden_size):
    230.         V, H = vocab_size, hidden_size
    231.  
    232.         # 初始化权重
    233.         W_in = 0.01 * np.random.randn(V, H).astype('f')
    234.         W_out = 0.01 * np.random.randn(H, V).astype('f')
    235.  
    236.         # 生成层
    237.         self.in_layer0 = MatMul(W_in)
    238.         self.in_layer1 = MatMul(W_in)
    239.         self.out_layer = MatMul(W_out)
    240.         self.loss_layer = SoftmaxWithLoss()
    241.  
    242.         # 将所有的权重和梯度整理到列表中
    243.         layers = [self.in_layer0, self.in_layer1, self.out_layer]
    244.         self.params, self.grads = [], []
    245.         for layer in layers:
    246.             self.params += layer.params
    247.             self.grads += layer.grads
    248.  
    249.         # 将单词的分布式表示设置为成员变量
    250.         self.word_vecs = W_in
    251.  
    252.     def forward(self, contexts, target):
    253.         h0 = self.in_layer0.forward(contexts[:, 0])
    254.         h1 = self.in_layer1.forward(contexts[:, 1])
    255.         h = (h0 + h1) * 0.5
    256.         score = self.out_layer.forward(h)
    257.         loss = self.loss_layer.forward(score, target)
    258.         return loss
    259.  
    260.     def backward(self, dout=1):
    261.         ds = self.loss_layer.backward(dout)
    262.         da = self.out_layer.backward(ds)
    263.         da *= 0.5
    264.         self.in_layer1.backward(da)
    265.         self.in_layer0.backward(da)
    266.         return None
    267.  
    268.  
    269. class Adam:
    270.     '''
    271.     Adam (http://arxiv.org/abs/1412.6980v8)
    272.     '''
    273.  
    274.     def __init__(self, lr=0.001, beta1=0.9, beta2=0.999):
    275.         self.lr = lr
    276.         self.beta1 = beta1
    277.         self.beta2 = beta2
    278.         self.iter = 0
    279.         self.m = None
    280.         self.v = None
    281.  
    282.     def update(self, params, grads):
    283.         if self.m is None:
    284.             self.m, self.v = [], []
    285.             for param in params:
    286.                 self.m.append(np.zeros_like(param))
    287.                 self.v.append(np.zeros_like(param))
    288.  
    289.         self.iter += 1
    290.         lr_t = self.lr * np.sqrt(1.0 - self.beta2 ** self.iter) / (1.0 - self.beta1 ** self.iter)
    291.  
    292.         for i in range(len(params)):
    293.             self.m[i] += (1 - self.beta1) * (grads[i] - self.m[i])
    294.             self.v[i] += (1 - self.beta2) * (grads[i] ** 2 - self.v[i])
    295.  
    296.             params[i] -= lr_t * self.m[i] / (np.sqrt(self.v[i]) + 1e-7)
    297.  
    298.  
    299. windows_size = 1
    300. hidden_size = 5
    301. batch_size = 3
    302. max_epoch = 1000
    303.  
    304. text = "You say goodbye and I say hello."
    305. corpus, word_to_id, id_to_word = preprocess(text)
    306. vocab_size = len(word_to_id)
    307. contexts, target = create_conext_target(corpus, windows_size)
    308.  
    309. target = convert_one_hot(target, vocab_size)
    310. contexts = convert_one_hot(contexts, vocab_size)
    311.  
    312. model =SimpleCBOW(vocab_size, hidden_size)
    313. optimizer = Adam()
    314. trainer = Trainer(model, optimizer)
    315.  
    316. trainer.fit(contexts, target, max_epoch, batch_size)
    317. trainer.plot()
    318.  
    319. word_vecs = model.word_vecs
    320. for word_id, word in id_to_word.items():
    321.     print(word, word_vecs[word_id])

            训练结果：

             单词的分布式表示：

     

  • 相关阅读:
    如何识别图片文字？这几个识别图片文字软件简单又高效
    BoW - Bag of Words - 词袋模型
    使用riscv-tests进行指令测试（二）
    2022-04-03 排查问题要知识沉淀
    Node.js内置模块
    使用GPT-4训练数据微调GPT-3.5 RAG管道
    【java】基本数据类型、包装类、string之间的转换
    力扣刷题学习SQL篇——1-10 选择（丢失信息的雇员——合并两个表的相同列union all）
    【c++】搜索二叉树的模拟实现
    EWM 过账期间修改（Posting only possible in periods***）
  • 原文地址：https://blog.csdn.net/qq_55202378/article/details/127676652