• transformer算法嵌入Embedding示例

    本示例的目的，是希望把sku训练好的embedding值嵌入到transformer算法中，从而提高transformer在销量预测算法中的准确性。

    一、训练数据格式说明

    1、embedding训练的数据格式示例：

    133657,本田#第八代雅阁,1816,4

    字段1表示：sku_id

    字段2表示：车型 # 款式

    字段3表示：车型 # 款式对应的序号id

    字段4表示：sku_id对应的类目信息

    2、销量预测训练的数据格式示例：

    0053#031188,0_0_0_0_0_0_1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_1_0_0_0_0_0_0_0_0_0_0_0_1_0_0_0_0_0_1_0_0_0_0_0_0_0_0_1_0_0_0_0_0_0_0_0_0_0_0_1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0
    字段1表示：门店code # sku_id

    字段2表示：历史N周每周的销量值

    二、Embedding Model阶段

    1、输入阶段

    （1）<sku_id, car_id>的组合关系，一个sku可以对应多个car_id

    （2）构建sku_car_matrix，其中car_id对应位置标记为1，其余位置标记为0

    （3）sku_id到sku_car_matrix中dim=0维的序号值的映射，方便后续sku_id根据序号值直接取到sku的embedding值，并持久化该映射关系

    （4）将sku_car_matrix装入DataSet中，其中__getitem()__为<train_data[index], train_data[index]>的组合，因为我们通过sku_car_matrix -> embedding -> sku_car_matrix从而获取embedding值。

    （5）将DataSet的值装入dataloader中，并设置shuffle为False，默认情况下DataLoader是会将数据打乱的。

    2、模型训练阶段

    （1）encoder - decoder网络架构

    （2）loss值

    decoder阶段输出的值与label值计算MseLoss值

    （3）每一批次迭代时会将中间值embedding保存下来，汇总所有迭代的embedding值便得到每一次epoch的embedding值。根据train_loss值，取最少值时的embedding值为最佳embedding值，并持久化该值

    （4）测试embedding效果的方法，将几个sku查看两两间的embedding的欧氏距离，理论上相近的sku欧氏距离值更小。

    （5）代码实现如下（embedding_model_train.py文件）：

    1. import os
    2. import numpy as np
    3. import pandas as pd
    4. from torch.utils.data import Dataset,DataLoader
    5. import torch
    6. import torch.nn as nn
    7. import logging
    8. from tqdm import trange
    9. import transformer_utils
    10.  
    11. logger = logging.getLogger('Transformer.Embedding')
    12.  
    13. class EmbeddingTrainDataset(Dataset):
    14.     def __init__(self, matrix_data):
    15.         self.train_data = matrix_data
    16.         self.train_len = len(matrix_data)
    17.  
    18.     def __len__(self):
    19.         return self.train_len
    20.  
    21.     def __getitem__(self, index):
    22.         return self.train_data[index], self.train_data[index]
    23.  
    24.  
    25. class AutoEncoder(nn.Module):
    26.     def __init__(self, input_dim, embedding_dim):
    27.         super(AutoEncoder, self).__init__()
    28.  
    29.  
    30.  
    31.         self.encoder = nn.Sequential(
    32.             nn.Linear(input_dim, input_dim // 2),
    33.             nn.Tanh(),
    34.             nn.Linear(input_dim // 2, input_dim // 4),
    35.             nn.Tanh(),
    36.             nn.Linear(input_dim // 4, embedding_dim),
    37.  
    38.         )
    39.         self.decoder = nn.Sequential(
    40.  
    41.             nn.Linear(embedding_dim, input_dim // 4),
    42.             nn.Tanh(),
    43.             nn.Linear(input_dim // 4, input_dim // 2),
    44.             nn.Tanh(),
    45.             nn.Linear(input_dim // 2, input_dim),
    46.         )
    47.  
    48.     def forward(self, x):
    49.         encoded = self.encoder(x)
    50.         decoded = self.decoder(encoded)
    51.         return encoded, decoded
    52.  
    53.  
    54. if __name__ == '__main__':
    55.     embedding_dim = 100
    56.     epochs = 10000
    57.     lr = 0.001
    58.     gamma = 0.95
    59.     batch_size = 1000
    60.  
    61.     transformer_utils.set_logger(os.path.join(os.getcwd(), 'train.log'))
    62.     data_frame = pd.read_csv(os.path.join(os.getcwd(), 'data', 'abs_sku_to_Car_classfication_onehot_detail.csv'), header=None,
    63.                              names=['sku_code', 'car_model', 'car_id', 'cat_id'], dtype={0: str, 1: str, 2: int, 3: int})
    64.     sku_code_set = set(data_frame['sku_code'].drop_duplicates())
    65.     sku2idx_dict = {}
    66.  
    67.     for i, sku_code in enumerate(sku_code_set):
    68.         sku2idx_dict[sku_code] = i
    69.     car_id_num = max(data_frame['car_id'])
    70.     sku_code_num = len(sku_code_set)
    71.     sku_code_car_matrix = np.zeros((sku_code_num, car_id_num), dtype='float32')
    72.     np.save(os.path.join(os.getcwd(), 'data', 'sku2idx_dict'), sku2idx_dict)
    73.  
    74.  
    75.  
    76.     for i in trange(len(data_frame)):
    77.         sku_code = data_frame.loc[i, 'sku_code']
    78.         car_id = data_frame.loc[i, 'car_id']
    79.  
    80.         sku_code_idx = sku2idx_dict[sku_code]
    81.         sku_code_car_matrix[sku_code_idx, car_id - 1] = 1
    82.  
    83.     train_set = EmbeddingTrainDataset(sku_code_car_matrix)
    84.     train_loader = DataLoader(train_set, batch_size=batch_size, num_workers=0, shuffle=False)
    85.  
    86.     device = "cuda" if torch.cuda.is_available() else "cpu"
    87.     autoencoder_model = AutoEncoder(car_id_num, embedding_dim).to(device)
    88.     criterion = nn.MSELoss()
    89.     optimizer = torch.optim.AdamW(autoencoder_model.parameters(), lr=lr)
    90.  
    91.     train_loss_summary = np.zeros(epochs)
    92.     best_evaluate_loss = 100.0
    93.     for epoch in trange(epochs):
    94.  
    95.         train_total_loss = 0
    96.         sku_encoder_embedding = np.zeros((sku_code_num, embedding_dim), dtype='float32')
    97.         train_loader_len = len(train_loader)
    98.  
    99.         for i, (x_input, x_label) in enumerate(train_loader):
    100.             x_input = x_input.to(device)
    101.             x_label = x_label.to(device)
    102.  
    103.             encoded, decoded = autoencoder_model(x_input)
    104.             loss = criterion(decoded, x_label)
    105.             optimizer.zero_grad()
    106.             loss.backward()
    107.             optimizer.step()
    108.             train_total_loss += loss.item()
    109.  
    110.             sku_encoder_embedding[(i * batch_size) : (i * batch_size + x_input.shape[0])] = encoded.detach().to('cpu').numpy()
    111.  
    112.         train_avg_loss = train_total_loss / train_loader_len
    113.         logger.info(f'epoch: {epoch + 1}, train_loss: {train_avg_loss}')
    114.  
    115.         is_best = False
    116.         if train_avg_loss < best_evaluate_loss:
    117.             is_best = True
    118.             best_evaluate_loss = train_avg_loss
    119.             np.save(os.path.join(os.getcwd(), 'data', 'sku2embedding'), sku_encoder_embedding)
    120.             logger.info(f'best embedding at: {epoch + 1}')
    121.  
    122.         if epoch >= 10:  # 太前面的去掉，免得影响后面曲线的可观测性
    123.             train_loss_summary[epoch] = train_avg_loss
    124.         if epoch % 10 == 1:
    125.             transformer_utils.plot_all_epoch(train_loss_summary, train_loss_summary, epoch, 'embedding_train_loss_summary.png')
    126.  
    127.     print('finish!')

     

    三、Embedding嵌入Transformer预测

    1、transformer预测数据预处理代码（transformer_preprocess_data.py文件）：

    1. import os
    2. import numpy as np
    3. import pandas as pd
    4. from tqdm import trange
    5.  
    6.  
    7. # 数据格式转换成标准格式
    8. def normalize_data_format(data):
    9.     data_sale_list_series = data['sale_info'].apply(lambda row: list(map(float, row.split("_"))))
    10.     data_frame = pd.DataFrame(item for item in data_sale_list_series)
    11.     data_frame = pd.concat((data['warehouse_sku'], data_frame), axis=1)
    12.     data_frame = data_frame.transpose()
    13.     return data_frame
    14.  
    15. # 平滑过大的值
    16. def smooth_big_value(data_frame):
    17.     columns_len = len(data_frame.columns)
    18.  
    19.     print(">>>>smooth_big_value")
    20.     for i in trange(columns_len):
    21.         values = data_frame.iloc[1:,i]
    22.         value_mean = np.mean(values[values > 0])
    23.         value_std = np.std(values[values > 0], ddof=1)
    24.         value_std = value_std if value_std > 0 else 0
    25.         values_new = np.round(np.where(values > value_mean + 3 * value_std, value_mean + 3 * value_std, values).astype(float))
    26.         values_new = np.array(values_new, dtype=np.int).astype(str)
    27.         data_frame.iloc[1:, i] = values_new
    28.  
    29.     return data_frame
    30.  
    31. # 获取列名和id之间的映射关系
    32. def gen_col2series(columns):
    33.     columns = columns.values[0,:]
    34.     id2series_dict = {}
    35.     series2id_dict = {}
    36.  
    37.     j = 0
    38.     for i, column in enumerate(columns):
    39.         id2series_dict[i] = column
    40.         if series2id_dict.get(column) is None:
    41.             series2id_dict[column] = j
    42.             j += 1
    43.  
    44.     return id2series_dict, series2id_dict
    45.  
    46. # 每列的最大值
    47. def gen_series2maxValue(data_frame):
    48.     series_max_value = np.max(data_frame[1:], axis=0)
    49.     series2maxValue = series_max_value.to_dict()
    50.     return series2maxValue
    51.  
    52. # 处理数据
    53. def prep_data(data, series2maxValue):
    54.     num_series = data.shape[1]
    55.     time_len = data.shape[0]
    56.  
    57.     windows_per_series = np.full((num_series), (time_len - backcast_len))
    58.     total_windows = np.sum(windows_per_series)
    59.     x_input = np.zeros((total_windows, backcast_len, 1 + 2), dtype='float32') # sale_info + series_info + max_value
    60.     label = np.zeros((total_windows, backcast_len), dtype='float32')
    61.  
    62.     print(">>>>prep_data")
    63.     count = 0
    64.     zero_count = 0
    65.     for series_idx in trange(num_series):
    66.         for i in range(windows_per_series[series_idx]):
    67.             x_input_data = data[i : i + backcast_len, series_idx]
    68.             x_input_series = series_idx
    69.             label_data = data[i + 1 : i + backcast_len + 1, series_idx]
    70.  
    71.             if np.max(x_input_data) > 0:
    72.                 x_input[count, :, 0] = x_input_data
    73.                 x_input[count, :, 1] = x_input_series
    74.                 x_input[count, :, 2] = series2maxValue.get(series_idx)
    75.                 label[count] = label_data
    76.                 x_input[count, :, 0] = x_input[count, :, 0] / series2maxValue.get(series_idx)
    77.                 label[count] = label[count] / series2maxValue.get(series_idx)
    78.                 count += 1
    79.             elif np.max(label_data) == 0 and zero_count < 2000 and np.random.choice([0,1], p=[0.6, 0.4]) > 0:
    80.                 x_input[count, :, 0] = x_input_data
    81.                 x_input[count, :, 1] = x_input_series
    82.                 x_input[count, :, 2] = 0
    83.                 label[count] = label_data
    84.                 zero_count += 1
    85.                 count += 1
    86.  
    87.     x_input = x_input[:count]
    88.     label = label[:count]
    89.     return x_input, label
    90.  
    91. # 切分测试集、验证集
    92. def split_train_test_data(x_input, label, train_ratio=0.8):
    93.     x_len = x_input.shape[0]
    94.     shuffle_idx = np.random.permutation(x_len)
    95.     train_x_len = int(x_len * train_ratio)
    96.     train_shuffle_idx = shuffle_idx[:train_x_len]
    97.     test_shuffle_idx = shuffle_idx[train_x_len:]
    98.  
    99.     train_x_input = x_input[train_shuffle_idx]
    100.     train_label = label[train_shuffle_idx]
    101.     test_x_input = x_input[test_shuffle_idx]
    102.     test_label = label[test_shuffle_idx]
    103.  
    104.     return train_x_input, train_label, test_x_input, test_label
    105.  
    106. if __name__ == '__main__':
    107.     backcast_len = 12
    108.     train_val_num = 110
    109.  
    110.     data_frame = pd.read_csv(os.path.join(os.getcwd(), 'data', 'ads_hub_sale_num_detail_simple.csv'), header=None, names=['warehouse_sku', 'sale_info'])
    111.     data_frame = normalize_data_format(data_frame)
    112.     data_frame = data_frame[:train_val_num]
    113.     data_frame = smooth_big_value(data_frame)
    114.  
    115.     id2series, series2id = gen_col2series(data_frame)
    116.     series2maxValue = gen_series2maxValue(data_frame)
    117.  
    118.     x_input, label = prep_data(data_frame.values[1:].astype('float'), series2maxValue)
    119.     train_x_input, train_label, test_x_input, test_label = split_train_test_data(x_input, label)
    120.  
    121.     np.save(os.path.join(os.getcwd(), 'data', 'train_data'), train_x_input)
    122.     np.save(os.path.join(os.getcwd(), 'data', 'train_label'), train_label)
    123.     np.save(os.path.join(os.getcwd(), 'data', 'test_data'), test_x_input)
    124.     np.save(os.path.join(os.getcwd(), 'data', 'test_label'), test_label)
    125.     np.save(os.path.join(os.getcwd(), 'data', 'series_max_value'), series2maxValue)
    126.     np.save(os.path.join(os.getcwd(), 'data', 'series2id'), series2id)
    127.     np.save(os.path.join(os.getcwd(), 'data', 'id2series'), id2series)
    128.  
    129.     print('finish!')

    2、dataloader加载数据的代码实现（transformer_dataloader.py文件）：

    1. import logging
    2. import os
    3. import numpy as np
    4. from torch.utils.data import Dataset
    5.  
    6. logger = logging.getLogger('Transformer.Data')
    7.  
    8. class TrainDataset(Dataset):
    9.     def __init__(self, data_path):
    10.         self.data = np.load(os.path.join(data_path, 'data', 'train_data.npy'))
    11.         self.label = np.load(os.path.join(data_path, 'data', 'train_label.npy'))
    12.         self.id2series_dict = np.load(os.path.join(data_path, 'data', 'id2series.npy')).item()
    13.         self.sku2idx_dict = np.load(os.path.join(data_path, 'data', 'sku2idx_dict.npy')).item()
    14.         self.sku2embedding = np.load(os.path.join(data_path, 'data', 'sku2embedding.npy'))
    15.         self.sku_embedding_avg = self.sku2embedding.mean(axis=0)
    16.         self.train_len = self.data.shape[0]
    17.         logger.info(f'train_len:{self.train_len}')
    18.         logger.info('building datasets from train_data.npy')
    19.  
    20.     def __len__(self):
    21.         return self.train_len
    22.  
    23.     def __getitem__(self, index):
    24.         series_idx = int(self.data[index,0,-2])
    25.         series = self.id2series_dict.get(series_idx)
    26.         sku_code = series.split('#')[1]
    27.         sku_idx = self.sku2idx_dict.get(sku_code)
    28.         if sku_idx is None:
    29.             sku_embedding = self.sku_embedding_avg
    30.         else:
    31.             sku_embedding = self.sku2embedding[sku_idx]
    32.  
    33.         return (self.data[index,:,:-2], series_idx, sku_embedding, self.label[index])
    34.  
    35. class TestDataset(Dataset):
    36.     def __init__(self, data_path):
    37.         self.data = np.load(os.path.join(data_path, 'data', 'test_data.npy'))
    38.         self.label = np.load(os.path.join(data_path, 'data', 'test_label.npy'))
    39.         self.id2series_dict = np.load(os.path.join(data_path, 'data', 'id2series.npy')).item()
    40.         self.sku2idx_dict = np.load(os.path.join(data_path, 'data', 'sku2idx_dict.npy')).item()
    41.         self.sku2embedding = np.load(os.path.join(data_path, 'data', 'sku2embedding.npy'))
    42.         self.sku_embedding_avg = self.sku2embedding.mean(axis=0)
    43.         self.test_len = self.data.shape[0]
    44.         logger.info(f'test_len:{self.test_len}')
    45.         logger.info('building datasets from test_data.npy')
    46.  
    47.     def __len__(self):
    48.         return self.test_len
    49.  
    50.     def __getitem__(self, index):
    51.         series_idx = int(self.data[index, 0, -2])
    52.         series = self.id2series_dict.get(series_idx)
    53.         sku_code = series.split('#')[1]
    54.         sku_idx = self.sku2idx_dict.get(sku_code)
    55.         if sku_idx is None:
    56.             sku_embedding = self.sku_embedding_avg
    57.         else:
    58.             sku_embedding = self.sku2embedding[sku_idx]
    59.  
    60.         return (self.data[index,:,:-2], series_idx, sku_embedding, self.data[index,0,-1], self.label[index])

    （1）id2series表示sku历史销量到sku_id的映射

    （2）sku2idx表示sku_id到embedding序号的映射

    （3）sku2embedding表示embedding序号到embedding值的映射

    （4）在dataset的__getitem__()函数中，先找到sku_id信息，并继续找到embedding值

    3、使用embedding值的流程

    代码实现如下（transformer_train.py文件）：

    1. import os
    2. import numpy as np
    3. import torch
    4. import torch.nn as nn
    5. import math
    6. import time
    7. import transformer_utils
    8. from transformer_dataloader import TrainDataset,TestDataset
    9. from torch.utils.data import DataLoader
    10. import logging
    11.  
    12. logger = logging.getLogger('Transformer.Train')
    13.  
    14.  
    15. class PositionalEncoding(nn.Module):
    16.     def __init__(self, d_model, max_len=5000):
    17.         super(PositionalEncoding, self).__init__()
    18.         pe = torch.zeros(max_len, d_model)
    19.         position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
    20.         div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
    21.         pe[:, 0::2] = torch.sin(position * div_term)
    22.         pe[:, 1::2] = torch.cos(position * div_term)
    23.         pe = pe.unsqueeze(0).transpose(0, 1)
    24.         self.register_buffer('pe', pe)
    25.  
    26.     def forward(self, x, embedding):
    27.         return x + self.pe[:x.size(0), :] + embedding
    28.  
    29.  
    30. class TransAm(nn.Module):
    31.     def __init__(self, feature_size=100, num_layers=1, dropout=0.1):
    32.         super(TransAm, self).__init__()
    33.         self.model_type = 'Transformer'
    34.         self.src_mask = None
    35.         self.pos_encoder = PositionalEncoding(feature_size)
    36.         self.encoder_layer = nn.TransformerEncoderLayer(d_model=feature_size, nhead=10, dropout=dropout)
    37.         self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)
    38.         self.decoder = nn.Linear(feature_size, 1)
    39.         self.init_weights()
    40.  
    41.     def init_weights(self):
    42.         initrange = 0.1
    43.         self.decoder.bias.data.zero_()
    44.         self.decoder.weight.data.uniform_(-initrange, initrange)
    45.  
    46.     def _generate_square_subsequent_mask(self, sz):
    47.         mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0,1)
    48.         mask = mask.float().masked_fill(mask==0, float('-inf')).masked_fill(mask == 1, float(0.0))
    49.         return mask
    50.  
    51.  
    52.     def forward(self, src, pre_embedding):
    53.         if self.src_mask is None or self.src_mask.shape[0] == len(src):
    54.             device = src.device
    55.             mask = self._generate_square_subsequent_mask(len(src)).to(device)
    56.             self.src_mask = mask
    57.             src = self.pos_encoder(src, pre_embedding)
    58.             output = self.transformer_encoder(src, self.src_mask)
    59.             output = self.decoder(output)
    60.             return output
    61.  
    62. def evaluate(model, test_loader):
    63.     test_total_loss = 0
    64.     test_total_with_max_loss = 0
    65.     model.eval()
    66.     test_loader_len = len(test_loader)
    67.     for i, (test_batch, idx, embedding_batch, max_value, labels) in enumerate(test_loader):
    68.         test_batch = test_batch.permute(1, 0, 2).to(device)
    69.         labels = labels.permute(1, 0).to(device)
    70.         embedding_batch = torch.unsqueeze(embedding_batch, dim=0).to(device)
    71.  
    72.         test_output = transformer_model(test_batch, embedding_batch)
    73.         test_output = torch.squeeze(test_output)
    74.         test_output[test_output < 0] = 0
    75.  
    76.         test_labels = labels[-1]
    77.         test_output = test_output[-1]
    78.         test_loss = criterion(test_output, test_labels)
    79.         test_total_loss += test_loss.item()
    80.  
    81.         max_value = max_value.to(device)
    82.         test_with_max_labels = test_labels * max_value
    83.         test_with_max_output = test_output * max_value
    84.         test_with_max_loss = criterion(test_with_max_output, test_with_max_labels)
    85.         test_total_with_max_loss += test_with_max_loss
    86.     test_avg_loss = test_total_loss / test_loader_len
    87.     test_with_max_avg_loss = test_total_with_max_loss / test_loader_len
    88.  
    89.     return test_avg_loss, test_with_max_avg_loss
    90.  
    91. if __name__ == '__main__':
    92.     transformer_utils.set_logger(os.path.join(os.getcwd(), 'train.log'))
    93.  
    94.     json_path = os.path.join(os.getcwd(), 'params.json')
    95.     params = transformer_utils.Params(json_path)
    96.     lr = params.lr
    97.     epochs = params.epochs
    98.     feature_size = params.feature_size
    99.     gamma = params.gamma
    100.     device = torch.device(params.mode)
    101.     input_window = params.input_window
    102.     feature_size = params.feature_size
    103.  
    104.  
    105.     train_set = TrainDataset(os.getcwd())
    106.     test_set = TestDataset(os.getcwd())
    107.     train_loader = DataLoader(train_set, batch_size=params.train_batch_size, num_workers=0)
    108.     test_loader = DataLoader(test_set, batch_size=params.test_batch_size, num_workers=0)
    109.  
    110.     transformer_model = TransAm(feature_size=feature_size).to(device)
    111.     criterion = nn.MSELoss()
    112.     optimizer = torch.optim.AdamW(transformer_model.parameters(), lr=lr)
    113.     scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=gamma)
    114.  
    115.     train_loss_summary = np.zeros(epochs)
    116.     test_loss_summary = np.zeros(epochs)
    117.  
    118.     best_evaluate_loss = 100.0
    119.     for epoch in range(1, epochs + 1):
    120.         epoch_start_time = time.time()
    121.         train_loader_len = len(train_loader)
    122.  
    123.         train_total_loss = 0
    124.         transformer_model.train()
    125.         for i, (train_batch, idx, embedding_batch, label_batch) in enumerate(train_loader):
    126.             optimizer.zero_grad()
    127.             train_batch = train_batch.permute(1, 0, 2).to(device)
    128.             label_batch = label_batch.permute(1, 0).to(device)
    129.             embedding_batch = torch.unsqueeze(embedding_batch, dim=0).to(device)
    130.  
    131.             output = transformer_model(train_batch, embedding_batch)
    132.             output = torch.squeeze(output)
    133.  
    134.             loss = criterion(output, label_batch)
    135.             loss.backward()
    136.             optimizer.step()
    137.             train_total_loss += loss.item()
    138.         train_avg_loss = train_total_loss / train_loader_len
    139.  
    140.         test_avg_loss, test_with_max_avg_loss = evaluate(transformer_model, test_loader)
    141.         logger.info(f'epoch: {epoch}, train_loss: {train_avg_loss}, test_loss: {test_avg_loss}, test_max_loss: {test_with_max_avg_loss}')
    142.  
    143.         is_best = False
    144.         if test_avg_loss < best_evaluate_loss:
    145.             is_best = True
    146.             best_evaluate_loss = test_avg_loss
    147.             transformer_utils.save_checkpoint({'epoch': epoch,
    148.                                               'state_dict': transformer_model.state_dict(),
    149.                                               'optim_dict': optimizer.state_dict()},
    150.                                              is_best,
    151.                                              epoch=epoch)
    152.  
    153.         train_loss_summary[epoch] = train_avg_loss
    154.         test_loss_summary[epoch] = test_avg_loss
    155.  
    156.         if epoch % 20 == 1:
    157.             transformer_utils.plot_all_epoch(test_loss_summary, test_loss_summary, epoch, 'train_test_loss_summary.png')
    158.  
    159.  
    160.     print('finish!')

    （1）读取train_loader的embedding_batch，它的shape为[1200, 100]，1200为batch_size，100为embedding_size

    （2）torch.unsqueeze(embedding_batch, dim=0)操作，使得embedding_batch的shape为[1, 1200, 100]，分别对应sequence_length, batch_size, embedding_size，也对应transformer的输入的shape需求

    （3）positionalEncoding中，最后输出为：x + self.pe[:x.size(0), :] + embedding

    其中，x的shape为[12, 1200, 1]，因为历史销量为12周，embedding_size为1，即只有销量值。

    self.pe[:x.size(0), :]的shape为[12, 1, 100]，因为对应一个batch中的各个值，所以pe是通用的，因此通过torch的广播机制，两者相加后的shape变为[12, 1200, 100]。

    embedding的shape为[1, 1200, 100]，因为sku的embedding是固有属性，不随时间(seq_length)发生变化，所以和上一步值通过广播机制后相加，最终维度还是[12, 1200, 100]。

    4、工具类文件的代码实现（transformer_utils.py）：

    1. import logging
    2. import os
    3. import torch
    4. import json
    5. import numpy as np
    6. from tqdm import tqdm
    7. import matplotlib.pyplot as plt
    8.  
    9. logger = logging.getLogger('Transformer.Utils')
    10.  
    11. class Params:
    12.     '''
    13.     class that loads hyperparameters from a json file
    14.     Example:
    15.     params = Params(json_path)
    16.     print(params.learning_rate)
    17.     '''
    18.  
    19.     def __init__(self, json_path):
    20.         with open(json_path) as f:
    21.             params = json.load(f)
    22.             self.__dict__.update(params)
    23.  
    24.  
    25. def set_logger(log_path):
    26.     '''Set the logger to log info in terminal and file `log_path`.
    27.     In general, it is useful to have a logger so that every output to the terminal is saved
    28.     in a permanent file. Here we save it to `model_dir/train.log`.
    29.     Example:
    30.     logging.info('Starting training...')
    31.     Args:
    32.         log_path: (string) where to log
    33.     '''
    34.     _logger = logging.getLogger('Transformer')
    35.     _logger.setLevel(logging.INFO)
    36.  
    37.     fmt = logging.Formatter('[%(asctime)s] %(name)s: %(message)s', '%H:%M:%S')
    38.  
    39.     class TqdmHandler(logging.StreamHandler):
    40.         def __init__(self, formatter):
    41.             logging.StreamHandler.__init__(self)
    42.             self.setFormatter(formatter)
    43.  
    44.         def emit(self, record):
    45.             msg = self.format(record)
    46.             tqdm.write(msg)
    47.  
    48.     file_handler = logging.FileHandler(log_path)
    49.     file_handler.setFormatter(fmt)
    50.     _logger.addHandler(file_handler)
    51.     _logger.addHandler(TqdmHandler(fmt))
    52.  
    53.  
    54. def save_checkpoint(state, is_best, epoch, save_checkpoint=False, ins_name=-1):
    55.     '''Saves model and training parameters at checkpoint + 'last.pth.tar'. If is_best==True, also saves
    56.     checkpoint + 'best.pth.tar'
    57.     Args:
    58.         state: (dict) contains model's state_dict, may contain other keys such as epoch, optimizer state_dict
    59.         is_best: (bool) True if it is the best model seen till now
    60.         checkpoint: (string) folder where parameters are to be saved
    61.         ins_name: (int) instance index
    62.     '''
    63.     if save_checkpoint:
    64.         if ins_name == -1:
    65.             filepath = os.path.join('transformer-training-checkpoint', f'epoch_{epoch}.pth.tar')
    66.         else:
    67.             filepath = os.path.join('transformer-training-checkpoint', f'epoch_{epoch}_ins_{ins_name}.pth.tar')
    68.         if not os.path.exists('transformer-training-checkpoint'):
    69.             logger.info(f'Checkpoint Directory does not exist! Making directory transformer-training-checkpoint')
    70.             os.mkdir('transformer-training-checkpoint')
    71.         torch.save(state, filepath)
    72.         logger.info(f'Checkpoint saved to {filepath}')
    73.     if is_best:
    74.         torch.save(state, os.path.join(os.getcwd(), 'base_model', 'best.pth.tar'))
    75.         logger.info('Best checkpoint saved to best.pth.tar')
    76.  
    77.  
    78. def plot_all_epoch(train_loss_summary, test_loss_summary, num_samples, png_name):
    79.     x = np.arange(start=1, stop=num_samples + 1)
    80.     f = plt.figure()
    81.     plt.plot(x, train_loss_summary[:num_samples], label='train_loss', linestyle='--')
    82.     plt.plot(x, test_loss_summary[:num_samples], label='test_loss', linestyle='-')
    83.     f.savefig(os.path.join('base_model', png_name))
    84.     plt.close()

    5、配置文件(params.json)代码：

    1. {
    2.   "train_batch_size": 1200,
    3.   "test_batch_size":100,
    4.   "lr": 0.005,
    5.   "epochs": 1000,
    6.   "feature_size": 100,
    7.   "gamma": 0.95,
    8.   "input_window": 12,
    9.   "mode": "cuda"
    10. }

  • 相关阅读:
    大环境之下软件测试行业趋势能否上升？
    机器学习/人工智能的笔试面试题目——CNN相关问题总结
    对表单的操作说明》
    用于无功补偿的固定电容晶闸管控制反应器研究（Simulink）
    ssm+vue的培训机构运营管理系统（有报告）。Javaee项目，ssm vue前后端分离项目。
    是使用local_setup.bash 还是 setup.bash
    微信小程序生成海报工具Painter
    JavaScript数据类型有哪些？
    球谐函数在环境光照中的使用原理
    61张图，图解Spring事务，拆解底层源码
  • 原文地址：https://blog.csdn.net/benben044/article/details/125476272