关注 码龄 粉丝数 原力等级 -- 被采纳 被点赞 采纳率 @WzS 2024-06-17 15:25
采纳率: 9.1%
浏览 2 首页/
人工智能
/ ValueError: invalid literal for int() with base 10: 'CLS' 机器学习python深度学习
import math
import re
import numpy as np
import tensorflow as tf
from collections import Counter
# 数据路径
DATA_PATH = './poetry.txt'
# 单行诗最大长度
MAX_LEN = 64
# 禁用的字符，拥有以下符号的诗将被忽略
DISALLOWED_WORDS = ['（', '）', '(', ')', '__', '《', '》', '【', '】', '[', ']']
# 一首诗（一行）对应一个列表的元素
poetry = []
# 按行读取数据 poetry.txt
with open(DATA_PATH, 'r', encoding='utf-8') as f:
lines = f.readlines()
# 遍历处理每一条数据
for line in lines:
# 利用正则表达式拆分标题和内容
fields = re.split(r"[:：]", line)
# 跳过异常数据
if len(fields) != 2:
continue
# 得到诗词内容（后面不需要标题）
content = fields[1]
# 跳过内容过长的诗词
if len(content) > MAX_LEN - 2:
continue
# 跳过存在禁用符的诗词
if any(word in content for word in DISALLOWED_WORDS):
continue
poetry.append(content.replace('\n', '')) # 最后要记得删除换行符
# 最小词频
MIN_WORD_FREQUENCY = 8
# 统计词频，利用Counter可以直接按单个字符进行统计词频
counter = Counter()
for line in poetry:
counter.update(line)
# 过滤掉低词频的词
tokens = [token for token, count in counter.items() if count >= MIN_WORD_FREQUENCY]
# 补上特殊词标记：填充字符标记、未知词标记、开始标记、结束标记
tokens = ["[PAD]", "[NONE]", "[START]", "[END]"] + tokens
# 映射: 词 -> 编号
word_idx = {}
# 映射: 编号 -> 词
idx_word = {}
for idx, word in enumerate(tokens):
word_idx[word] = idx
idx_word[idx] = word
#分词器
class Tokenizer:
"""
分词器
"""
def __init__(self, tokens):
# 词汇表大小
self.dict_size = len(tokens)
# 生成映射关系
self.token_id = {} # 映射: 词 -> 编号
self.id_token = {} # 映射: 编号 -> 词
for idx, word in enumerate(tokens):
self.token_id[word] = idx
self.id_token[idx] = word
# 各个特殊标记的编号id，方便其他地方使用
self.start_id = self.token_id["[START]"]
self.end_id = self.token_id["[END]"]
self.none_id = self.token_id["[NONE]"]
self.pad_id = self.token_id["[PAD]"]
def id_to_token(self, token_id):
"""
编号 -> 词
"""
return self.id_token.get(token_id)
def token_to_id(self, token):
"""
词 -> 编号
"""
return self.token_id.get(token, self.none_id)
def encode(self, tokens):
"""
词列表 -> [START]编号 + 编号列表 + [END]编号
"""
token_ids = [self.start_id, ] # 起始标记
# 遍历，词转编号
for token in tokens:
token_ids.append(self.token_to_id(token))
token_ids.append(self.end_id) # 结束标记
return token_ids
def decode(self, token_ids):
"""
编号列表 -> 词列表(去掉起始、结束标记)
"""
# 起始、结束标记
flag_tokens = {"[START]", "[END]"}
tokens = []
for idx in token_ids:
token = self.id_to_token(idx)
# 跳过起始、结束标记
if token not in flag_tokens:
tokens.append(token)
return tokens
tokenizer = Tokenizer(tokens)
#数据集生成器
class PoetryDataSet:
"""
古诗数据集生成器
"""
def __init__(self, data, tokenizer, batch_size):
# 数据集
self.data = data
self.total_size = len(self.data)
# 分词器，用于词转编号
self.tokenizer = tokenizer
# 每批数据量
self.batch_size = batch_size
# 每个epoch迭代的步数
self.steps = int(math.floor(len(self.data) / self.batch_size))
# 计算最大长度
self.max_length = max(map(len, data)) + 2 # 加2是为了包含[START]和[END]标记
# ... 其他方法保持不变 ...
def pad_line(self, line, padding=None):
"""
对齐单行数据
"""
if padding is None:
padding = self.tokenizer.pad_id
# 使用max_length进行填充
padding_length = self.max_length - len(line)
if padding_length > 0:
return line + [padding] * padding_length
else:
return line[:self.max_length]
def __len__(self):
return self.steps
def __iter__(self):
# 打乱数据
np.random.shuffle(self.data)
# 迭代一个epoch，每次yield一个batch
for start in range(0, self.total_size, self.batch_size):
end = min(start + self.batch_size, self.total_size)
data = self.data[start:end]
max_length = max(map(len, data))
batch_data = []
for str_line in data:
# 对每一行诗词进行编码、并补齐padding
encode_line = self.tokenizer.encode(str_line)
pad_encode_line = self.pad_line(encode_line, max_length + 2) # 加2是因为tokenizer.encode会添加START和END
batch_data.append(pad_encode_line)
batch_data = np.array(batch_data)
# yield 特征、标签
yield batch_data[:, :-1], batch_data[:, 1:]
def generator(self):
while True:
yield from self.__iter__()
BATCH_SIZE = 32
dataset = PoetryDataSet(poetry, tokenizer, BATCH_SIZE)
print(tokenizer.dict_size)
#模型的构建与训练
model = tf.keras.Sequential([
# 词嵌入层
tf.keras.layers.Embedding(input_dim=tokenizer.dict_size, output_dim=150),
# 第一个LSTM层
tf.keras.layers.LSTM(150, dropout=0.5, return_sequences=True),
# 第二个LSTM层
tf.keras.layers.LSTM(150, dropout=0.5, return_sequences=True),
# 利用TimeDistributed对每个时间步的输出都做Dense操作(softmax激活)
tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(tokenizer.dict_size, activation='softmax')),
])
model.compile(
optimizer=tf.keras.optimizers.Adam(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
)
model.summary()
# model.fit(
# dataset.generator(),
# steps_per_epoch=dataset.steps,
# epochs=1
# )
model.save("./rnn_model.h5")
#预测
def predict_and_update(model, tokenizer, s, token_ids, poetry, punctuation_ids):
# 5.1) 进行预测，只保留第一个样例(我们输入的样例数只有1)的、最后一个token的预测的输出
output = model(np.array([token_ids], dtype=np.int32))
_probas = output.numpy()[0, -1, 3:]
del output
# 5.2) 重新计算预测概率
p_args = _probas.argsort()[::-1][:100]
p = _probas[p_args]
p = p / sum(p)
# 5.3) 根据概率，随机选择一个词作为预测结果
target_index = np.random.choice(len(p), p=p)
target = p_args[target_index] + 3
# 5.4) 保存
token_ids.append(target)
if target > 3:
poetry.append(tokenizer.id_to_token(target))
if target in punctuation_ids:
return True # Indicates end of a line
return False # Continue generating text
# 调用示例
# 假设 tokenizer 和 model 已经定义好
tokenizer = Tokenizer(tokens)
model = tf.keras.models.load_model("./rnn_model.h5")
# 定义输入字符串 s 和初始 token_ids、poetry
s = "your_input_text"
token_ids = ['CLS']
poetry = []
# 定义标点符号的 token_ids
punctuation_ids = ["[PAD]", "[NONE]", "[START]", "[END]"] # 请替换成实际标点符号的 ID
# 循环生成诗句
while True:
if predict_and_update(model, tokenizer, s, token_ids, poetry, punctuation_ids):
break
# 输出生成的诗句
print("Generated Poetry:")
print(" ".join(poetry))
D:\Python\python.exe D:\Python学习专用\pythonProject\test2.py
2024-06-17 15:22:28.202401: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-06-17 15:22:29.171460: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
3434
2024-06-17 15:22:31.762222: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
Model: "sequential"
┌─────────────────────────────────┬────────────────────────┬───────────────┐
│ Layer (type) │ Output Shape │ Param # │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ embedding (Embedding) │ ? │ 0 (unbuilt) │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ lstm (LSTM) │ ? │ 0 (unbuilt) │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ lstm_1 (LSTM) │ ? │ 0 (unbuilt) │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ time_distributed │ ? │ 0 (unbuilt) │
│ (TimeDistributed) │ │ │
└─────────────────────────────────┴────────────────────────┴───────────────┘
Total params: 0 (0.00 B)
Trainable params: 0 (0.00 B)
Non-trainable params: 0 (0.00 B)
WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
Traceback (most recent call last):
File "D:\Python学习专用\pythonProject\test2.py", line 261, in
if predict_and_update(model, tokenizer, s, token_ids, poetry, punctuation_ids):
File "D:\Python学习专用\pythonProject\test2.py", line 222, in predict_and_update
output = model(np.array([token_ids], dtype=np.int32))
ValueError: invalid literal for int() with base 10: 'CLS'
进程已结束，退出代码为 1
展开全部
收起
写回答
好问题
0 提建议
追加酬金
关注问题
微信扫一扫 点击复制链接 分享 邀请回答
编辑 收藏 删除 结题 收藏 举报 追加酬金 (90%的用户在追加酬金后获得了解决方案) 当前问题酬金 ¥ 0
(可追加 ¥500）
¥ 15¥ 20¥ 50¥ 100¥ 200 支付方式 扫码支付
二维码出错
点击刷新
支付金额
15 元
提供问题酬金的用户不参与问题酬金结算和分配
支付即为同意
《付费问题酬金结算规则》 结题 再想想 删除 再等等
