• 【自然语言处理（NLP）实战】LSTM网络实现中文文本情感分析（手把手与教学超详细）

    目录

    引言：

    1.所有文件展示：

    1.中文停用词数据（hit_stopwords.txt)来源于：

    2.其中data数据集为chinese_text_cnn-master.zip提取出的文件。点击链接进入github，点击Code、Download ZIP即可下载。

    2.安装依赖库：

    3.数据预处理（data_set.py)：

    train.txt-去除停用词后的训练集文件：

    test.txt -去除停用词后的测试集文件：

    4. 模型训练以及保存（main.py)

    1.LSTM模型搭建：

    2.main.py代价展示 ：

     3.模型保存

    4.训练结果 

    5.LSTM模型测试（test.py) 

     1.测试结果：

    2.测试结果：

    6.完整代码展示：

    1.data_set.py

    2.mian.py

    3.test.py 

    引言：

    在当今数字化时代，人们在社交媒体、评论平台以及各类在线交流中产生了海量的文本数据。这些数据蕴含着丰富的情感信息，从而成为了深入理解用户态度、市场趋势，甚至社会情绪的宝贵资源。自然语言处理（NLP）的发展为我们提供了强大的工具，使得对文本情感进行分析成为可能。在这个领域中，长短时记忆网络（LSTM）凭借其能够捕捉文本序列中长距离依赖关系的能力，成为了情感分析任务中的一项重要技术。

    本篇博客将手把手地教你如何使用LSTM网络实现中文文本情感分析。我们将从数据预处理开始，逐步构建一个端到端的情感分析模型。通过详细的步骤和示例代码，深入了解如何处理中文文本数据、构建LSTM模型、进行训练和评估。

    1.所有文件展示：

    1.中文停用词数据（hit_stopwords.txt)来源于：

    项目目录预览 - stopwords - GitCode

    2.其中data数据集为chinese_text_cnn-master.zip提取出的文件。点击链接进入github，点击Code、Download ZIP即可下载。

    2.安装依赖库：

    1. pip install torch # 搭建LSTM模型
    2. pip install gensim # 中文文本词向量转换
    3. pip install numpy # 数据清洗、预处理
    4. pip install pandas

    3.数据预处理（data_set.py)：

    1. # -*- coding: utf-8 -*-
    2. # @Time : 2023/11/15 10:52
    3. # @Author :Muzi
    4. # @File : data_set.py
    5. # @Software: PyCharm
    6. import pandas as pd
    7. import jieba
    8.  
    9.  
    10. # 数据读取
    11. def load_tsv(file_path):
    12.     data = pd.read_csv(file_path, sep='\t')
    13.     data_x = data.iloc[:, -1]
    14.     data_y = data.iloc[:, 1]
    15.     return data_x, data_y
    16.  
    17. train_x, train_y = load_tsv("./data/train.tsv")
    18. test_x, test_y = load_tsv("./data/test.tsv")
    19. train_x=[list(jieba.cut(x)) for x in train_x]
    20. test_x=[list(jieba.cut(x)) for x in test_x]
    21.  
    22. with open('./hit_stopwords.txt','r',encoding='UTF8') as f:
    23.     stop_words=[word.strip() for word in f.readlines()]
    24.     print('Successfully')
    25. def drop_stopword(datas):
    26.     for data in datas:
    27.         for word in data:
    28.             if word in stop_words:
    29.                 data.remove(word)
    30.     return datas
    31.  
    32. def save_data(datax,path):
    33.     with open(path, 'w', encoding="UTF8") as f:
    34.         for lines in datax:
    35.             for i, line in enumerate(lines):
    36.                 f.write(str(line))
    37.                 # 如果不是最后一行，就添加一个逗号
    38.                 if i != len(lines) - 1:
    39.                     f.write(',')
    40.             f.write('\n')
    41.  
    42. if __name__ == '__main':
    43.     train_x=drop_stopword(train_x)
    44.     test_x=drop_stopword(test_x)
    45.  
    46.     save_data(train_x,'./train.txt')
    47.     save_data(test_x,'./test.txt')
    48.     print('Successfully')

    train.txt-去除停用词后的训练集文件：

     

    test.txt -去除停用词后的测试集文件：

    4. 模型训练以及保存（main.py)

    1.LSTM模型搭建：

    不同的数据集应该有不同的分类标准，我这里用到的数据模型属于二分类问题

    1. # 定义LSTM模型
    2. class LSTMModel(nn.Module):
    3.     def __init__(self, input_size, hidden_size, output_size):
    4.         super(LSTMModel, self).__init__()
    5.         self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True)
    6.         self.fc = nn.Linear(hidden_size, output_size)
    7.  
    8.     def forward(self, x):
    9.         lstm_out, _ = self.lstm(x)
    10.         output = self.fc(lstm_out[:, -1, :])  # 取序列的最后一个输出
    11.         return output
    12.  
    13. # 定义模型
    14. input_size = word2vec_model.vector_size
    15. hidden_size = 50  # 你可以根据需要调整隐藏层大小
    16. output_size = 2  # 输出的大小，根据你的任务而定
    17.  
    18. model = LSTMModel(input_size, hidden_size, output_size)
    19. # 定义损失函数和优化器
    20. criterion = nn.CrossEntropyLoss()  # 交叉熵损失函数
    21. optimizer = torch.optim.Adam(model.parameters(), lr=0.0002)

    2.main.py代价展示 ：

    1. # -*- coding: utf-8 -*-
    2. # @Time : 2023/11/13 20:31
    3. # @Author :Muzi
    4. # @File : mian.py.py
    5. # @Software: PyCharm
    6. import pandas as pd
    7. import torch
    8. from torch import nn
    9. import jieba
    10. from gensim.models import Word2Vec
    11. import numpy as np
    12. from data_set import load_tsv
    13. from torch.utils.data import DataLoader, TensorDataset
    14.  
    15.  
    16. # 数据读取
    17. def load_txt(path):
    18.     with open(path,'r',encoding='utf-8') as f:
    19.         data=[[line.strip()] for line in f.readlines()]
    20.         return data
    21.  
    22. train_x=load_txt('train.txt')
    23. test_x=load_txt('test.txt')
    24. train=train_x+test_x
    25. X_all=[i for x in train for i in x]
    26.  
    27. _, train_y = load_tsv("./data/train.tsv")
    28. _, test_y = load_tsv("./data/test.tsv")
    29. # 训练Word2Vec模型
    30. word2vec_model = Word2Vec(sentences=X_all, vector_size=100, window=5, min_count=1, workers=4)
    31.  
    32. # 将文本转换为Word2Vec向量表示
    33. def text_to_vector(text):
    34.     vector = [word2vec_model.wv[word] for word in text if word in word2vec_model.wv]
    35.     return sum(vector) / len(vector) if vector else [0] * word2vec_model.vector_size
    36.  
    37. X_train_w2v = [[text_to_vector(text)] for line in train_x for text in line]
    38. X_test_w2v = [[text_to_vector(text)] for line in test_x for text in line]
    39.  
    40. # 将词向量转换为PyTorch张量
    41. X_train_array = np.array(X_train_w2v, dtype=np.float32)
    42. X_train_tensor = torch.Tensor(X_train_array)
    43. X_test_array = np.array(X_test_w2v, dtype=np.float32)
    44. X_test_tensor = torch.Tensor(X_test_array)
    45. #使用DataLoader打包文件
    46. train_dataset = TensorDataset(X_train_tensor, torch.LongTensor(train_y))
    47. train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
    48. test_dataset = TensorDataset(X_test_tensor,torch.LongTensor(test_y))
    49. test_loader = DataLoader(test_dataset, batch_size=64, shuffle=True)
    50. # 定义LSTM模型
    51. class LSTMModel(nn.Module):
    52.     def __init__(self, input_size, hidden_size, output_size):
    53.         super(LSTMModel, self).__init__()
    54.         self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True)
    55.         self.fc = nn.Linear(hidden_size, output_size)
    56.  
    57.     def forward(self, x):
    58.         lstm_out, _ = self.lstm(x)
    59.         output = self.fc(lstm_out[:, -1, :])  # 取序列的最后一个输出
    60.         return output
    61.  
    62. # 定义模型
    63. input_size = word2vec_model.vector_size
    64. hidden_size = 50  # 你可以根据需要调整隐藏层大小
    65. output_size = 2  # 输出的大小，根据你的任务而定
    66.  
    67. model = LSTMModel(input_size, hidden_size, output_size)
    68. # 定义损失函数和优化器
    69. criterion = nn.CrossEntropyLoss()  # 交叉熵损失函数
    70. optimizer = torch.optim.Adam(model.parameters(), lr=0.0002)
    71.  
    72. if __name__ == "__main__":
    73.     # 训练模型
    74.     num_epochs = 10
    75.     log_interval = 100  # 每隔100个批次输出一次日志
    76.     loss_min=100
    77.     for epoch in range(num_epochs):
    78.         model.train()
    79.         for batch_idx, (data, target) in enumerate(train_loader):
    80.             outputs = model(data)
    81.             loss = criterion(outputs, target)
    82.  
    83.             optimizer.zero_grad()
    84.             loss.backward()
    85.             optimizer.step()
    86.  
    87.             if batch_idx % log_interval == 0:
    88.                 print('Epoch [{}/{}], Batch [{}/{}], Loss: {:.4f}'.format(
    89.                     epoch + 1, num_epochs, batch_idx, len(train_loader), loss.item()))
    90.             # 保存最佳模型
    91.             if loss.item()
    92.                 loss_min=loss.item()
    93.                 torch.save(model, 'model.pth')
    94.  
    95.     # 模型评估
    96.     with torch.no_grad():
    97.         model.eval()
    98.         correct = 0
    99.         total = 0
    100.         for data, target in test_loader:
    101.             outputs = model(data)
    102.             _, predicted = torch.max(outputs.data, 1)
    103.             total += target.size(0)
    104.             correct += (predicted == target).sum().item()
    105.  
    106.         accuracy = correct / total
    107.         print('Test Accuracy: {:.2%}'.format(accuracy))

     3.模型保存

    1.  # 保存最佳模型
    2.             if loss.item()
    3.                 loss_min=loss.item()
    4.                 torch.save(model, 'model.pth')

    4.训练结果 

    5.LSTM模型测试（test.py) 

    1. # -*- coding: utf-8 -*-
    2. # @Time : 2023/11/15 15:53
    3. # @Author :Muzi
    4. # @File : test.py.py
    5. # @Software: PyCharm
    6. import torch
    7. import jieba
    8. from torch import nn
    9. from gensim.models import Word2Vec
    10. import numpy as np
    11.  
    12. class LSTMModel(nn.Module):
    13.     def __init__(self, input_size, hidden_size, output_size):
    14.         super(LSTMModel, self).__init__()
    15.         self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True)
    16.         self.fc = nn.Linear(hidden_size, output_size)
    17.  
    18.     def forward(self, x):
    19.         lstm_out, _ = self.lstm(x)
    20.         output = self.fc(lstm_out[:, -1, :])  # 取序列的最后一个输出
    21.         return output
    22.  
    23. # 数据读取
    24. def load_txt(path):
    25.     with open(path,'r',encoding='utf-8') as f:
    26.         data=[[line.strip()] for line in f.readlines()]
    27.         return data
    28.  
    29. #去停用词
    30. def drop_stopword(datas):
    31.     # 假设你有一个函数用于预处理文本数据
    32.     with open('./hit_stopwords.txt', 'r', encoding='UTF8') as f:
    33.         stop_words = [word.strip() for word in f.readlines()]
    34.     datas=[x for x in datas if x not in stop_words]
    35.     return datas
    36.  
    37. def preprocess_text(text):
    38.     text=list(jieba.cut(text))
    39.     text=drop_stopword(text)
    40.     return text
    41.  
    42. # 将文本转换为Word2Vec向量表示
    43. def text_to_vector(text):
    44.     train_x = load_txt('train.txt')
    45.     test_x = load_txt('test.txt')
    46.     train = train_x + test_x
    47.     X_all = [i for x in train for i in x]
    48.     # 训练Word2Vec模型
    49.     word2vec_model = Word2Vec(sentences=X_all, vector_size=100, window=5, min_count=1, workers=4)
    50.     vector = [word2vec_model.wv[word] for word in text if word in word2vec_model.wv]
    51.     return sum(vector) / len(vector) if vector else [0] * word2vec_model.vector_size
    52.  
    53. if __name__ == '__main__':
    54.     # input_text = "这个车完全就是垃圾,又热又耗油"
    55.     input_text = "这个车我开了好几年，还是不错的"
    56.     label = {1: "正面情绪", 0: "负面情绪"}
    57.     model = torch.load('model.pth')
    58.     # 预处理输入数据
    59.     input_data = preprocess_text(input_text)
    60.     # 确保输入词向量与模型维度和数据类型相同
    61.     input_data=[[text_to_vector(input_data)]]
    62.     input_arry= np.array(input_data, dtype=np.float32)
    63.     input_tensor = torch.Tensor(input_arry)
    64.     # 将输入数据传入模型
    65.     with torch.no_grad():
    66.         output = model(input_tensor)
    67.     predicted_class = label[torch.argmax(output).item()]
    68.     print(f"predicted_text:{input_text}")
    69.     print(f"模型预测的类别: {predicted_class}")
    70.

     1.测试结果：

    2.测试结果：

    6.完整代码展示：

    1.data_set.py

    1. import pandas as pd
    2. import jieba
    3.  
    4. # 数据读取
    5. def load_tsv(file_path):
    6.     data = pd.read_csv(file_path, sep='\t')
    7.     data_x = data.iloc[:, -1]
    8.     data_y = data.iloc[:, 1]
    9.     return data_x, data_y
    10.  
    11. with open('./hit_stopwords.txt','r',encoding='UTF8') as f:
    12.     stop_words=[word.strip() for word in f.readlines()]
    13.     print('Successfully')
    14. def drop_stopword(datas):
    15.     for data in datas:
    16.         for word in data:
    17.             if word in stop_words:
    18.                 data.remove(word)
    19.     return datas
    20.  
    21. def save_data(datax,path):
    22.     with open(path, 'w', encoding="UTF8") as f:
    23.         for lines in datax:
    24.             for i, line in enumerate(lines):
    25.                 f.write(str(line))
    26.                 # 如果不是最后一行，就添加一个逗号
    27.                 if i != len(lines) - 1:
    28.                     f.write(',')
    29.             f.write('\n')
    30.  
    31. if __name__ == '__main':
    32.     train_x, train_y = load_tsv("./data/train.tsv")
    33.     test_x, test_y = load_tsv("./data/test.tsv")
    34.     train_x = [list(jieba.cut(x)) for x in train_x]
    35.     test_x = [list(jieba.cut(x)) for x in test_x]
    36.     train_x=drop_stopword(train_x)
    37.     test_x=drop_stopword(test_x)
    38.     save_data(train_x,'./train.txt')
    39.     save_data(test_x,'./test.txt')
    40.     print('Successfully')

    2.mian.py

    1. import pandas as pd
    2. import torch
    3. from torch import nn
    4. import jieba
    5. from gensim.models import Word2Vec
    6. import numpy as np
    7. from data_set import load_tsv
    8. from torch.utils.data import DataLoader, TensorDataset
    9.  
    10.  
    11. # 数据读取
    12. def load_txt(path):
    13.     with open(path,'r',encoding='utf-8') as f:
    14.         data=[[line.strip()] for line in f.readlines()]
    15.         return data
    16.  
    17. train_x=load_txt('train.txt')
    18. test_x=load_txt('test.txt')
    19. train=train_x+test_x
    20. X_all=[i for x in train for i in x]
    21.  
    22. _, train_y = load_tsv("./data/train.tsv")
    23. _, test_y = load_tsv("./data/test.tsv")
    24. # 训练Word2Vec模型
    25. word2vec_model = Word2Vec(sentences=X_all, vector_size=100, window=5, min_count=1, workers=4)
    26.  
    27. # 将文本转换为Word2Vec向量表示
    28. def text_to_vector(text):
    29.     vector = [word2vec_model.wv[word] for word in text if word in word2vec_model.wv]
    30.     return sum(vector) / len(vector) if vector else [0] * word2vec_model.vector_size
    31.  
    32. X_train_w2v = [[text_to_vector(text)] for line in train_x for text in line]
    33. X_test_w2v = [[text_to_vector(text)] for line in test_x for text in line]
    34.  
    35. # 将词向量转换为PyTorch张量
    36. X_train_array = np.array(X_train_w2v, dtype=np.float32)
    37. X_train_tensor = torch.Tensor(X_train_array)
    38. X_test_array = np.array(X_test_w2v, dtype=np.float32)
    39. X_test_tensor = torch.Tensor(X_test_array)
    40. #使用DataLoader打包文件
    41. train_dataset = TensorDataset(X_train_tensor, torch.LongTensor(train_y))
    42. train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
    43. test_dataset = TensorDataset(X_test_tensor,torch.LongTensor(test_y))
    44. test_loader = DataLoader(test_dataset, batch_size=64, shuffle=True)
    45. # 定义LSTM模型
    46. class LSTMModel(nn.Module):
    47.     def __init__(self, input_size, hidden_size, output_size):
    48.         super(LSTMModel, self).__init__()
    49.         self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True)
    50.         self.fc = nn.Linear(hidden_size, output_size)
    51.  
    52.     def forward(self, x):
    53.         lstm_out, _ = self.lstm(x)
    54.         output = self.fc(lstm_out[:, -1, :])  # 取序列的最后一个输出
    55.         return output
    56.  
    57. # 定义模型
    58. input_size = word2vec_model.vector_size
    59. hidden_size = 50  # 你可以根据需要调整隐藏层大小
    60. output_size = 2  # 输出的大小，根据你的任务而定
    61.  
    62. model = LSTMModel(input_size, hidden_size, output_size)
    63. # 定义损失函数和优化器
    64. criterion = nn.CrossEntropyLoss()  # 交叉熵损失函数
    65. optimizer = torch.optim.Adam(model.parameters(), lr=0.0002)
    66.  
    67. if __name__ == "__main__":
    68.     # 训练模型
    69.     num_epochs = 10
    70.     log_interval = 100  # 每隔100个批次输出一次日志
    71.     loss_min=100
    72.     for epoch in range(num_epochs):
    73.         model.train()
    74.         for batch_idx, (data, target) in enumerate(train_loader):
    75.             outputs = model(data)
    76.             loss = criterion(outputs, target)
    77.  
    78.             optimizer.zero_grad()
    79.             loss.backward()
    80.             optimizer.step()
    81.  
    82.             if batch_idx % log_interval == 0:
    83.                 print('Epoch [{}/{}], Batch [{}/{}], Loss: {:.4f}'.format(
    84.                     epoch + 1, num_epochs, batch_idx, len(train_loader), loss.item()))
    85.             # 保存最佳模型
    86.             if loss.item()
    87.                 loss_min=loss.item()
    88.                 torch.save(model, 'model.pth')
    89.  
    90.     # 模型评估
    91.     with torch.no_grad():
    92.         model.eval()
    93.         correct = 0
    94.         total = 0
    95.         for data, target in test_loader:
    96.             outputs = model(data)
    97.             _, predicted = torch.max(outputs.data, 1)
    98.             total += target.size(0)
    99.             correct += (predicted == target).sum().item()
    100.  
    101.         accuracy = correct / total
    102.         print('Test Accuracy: {:.2%}'.format(accuracy))

    3.test.py 

    1. import torch
    2. import jieba
    3. from torch import nn
    4. from gensim.models import Word2Vec
    5. import numpy as np
    6.  
    7. class LSTMModel(nn.Module):
    8.     def __init__(self, input_size, hidden_size, output_size):
    9.         super(LSTMModel, self).__init__()
    10.         self.lstm = nn.LSTM(input_size, hidden_size, batch_first=True)
    11.         self.fc = nn.Linear(hidden_size, output_size)
    12.  
    13.     def forward(self, x):
    14.         lstm_out, _ = self.lstm(x)
    15.         output = self.fc(lstm_out[:, -1, :])  # 取序列的最后一个输出
    16.         return output
    17.  
    18. # 数据读取
    19. def load_txt(path):
    20.     with open(path,'r',encoding='utf-8') as f:
    21.         data=[[line.strip()] for line in f.readlines()]
    22.         return data
    23.  
    24. #去停用词
    25. def drop_stopword(datas):
    26.     # 假设你有一个函数用于预处理文本数据
    27.     with open('./hit_stopwords.txt', 'r', encoding='UTF8') as f:
    28.         stop_words = [word.strip() for word in f.readlines()]
    29.     datas=[x for x in datas if x not in stop_words]
    30.     return datas
    31.  
    32. def preprocess_text(text):
    33.     text=list(jieba.cut(text))
    34.     text=drop_stopword(text)
    35.     return text
    36.  
    37. # 将文本转换为Word2Vec向量表示
    38. def text_to_vector(text):
    39.     train_x = load_txt('train.txt')
    40.     test_x = load_txt('test.txt')
    41.     train = train_x + test_x
    42.     X_all = [i for x in train for i in x]
    43.     # 训练Word2Vec模型
    44.     word2vec_model = Word2Vec(sentences=X_all, vector_size=100, window=5, min_count=1, workers=4)
    45.     vector = [word2vec_model.wv[word] for word in text if word in word2vec_model.wv]
    46.     return sum(vector) / len(vector) if vector else [0] * word2vec_model.vector_size
    47.  
    48. if __name__ == '__main__':
    49.     input_text = "这个车完全就是垃圾,又热又耗油"
    50.     # input_text = "这个车我开了好几年，还是不错的"
    51.     label = {1: "正面情绪", 0: "负面情绪"}
    52.     model = torch.load('model.pth')
    53.     # 预处理输入数据
    54.     input_data = preprocess_text(input_text)
    55.     # 确保输入词向量与模型维度和数据类型相同
    56.     input_data=[[text_to_vector(input_data)]]
    57.     input_arry= np.array(input_data, dtype=np.float32)
    58.     input_tensor = torch.Tensor(input_arry)
    59.     # 将输入数据传入模型
    60.     with torch.no_grad():
    61.         output = model(input_tensor)
    62.     # 这里只一个简单的示例
    63.     predicted_class = label[torch.argmax(output).item()]
    64.     print(f"predicted_text:{input_text}")
    65.     print(f"模型预测的类别: {predicted_class}")

  • 相关阅读:
    Electron自动化测试技术选型调研
    正点原子lwIP学习笔记——Socket接口UDP实验
    企业搭建网站用哪种服务器
    PyQt5如何在Qtdesigner里修改按钮形状、大小、按钮颜色、字体颜色等参数，尤其是如何将按钮修改成圆形。
    在deepin上使用Fleet开发SpringBoot 3.0.0项目
    【高等数学基础进阶】定积分与反常积分-定积分
    LeetCode 面试题 03.04. 化栈为队
    华为欧拉系统安装
    使用naive-ui做一个标签页展示列表
    Python 模拟Laguerre Polynomial拉盖尔多项式
  • 原文地址：https://blog.csdn.net/m0_74053536/article/details/134379831