• PyTorch搭建RNN联合嵌入模型（LSTM GRU）实现视觉问答（VQA）实战（超详细 附数据集和源码）

    需要源码和数据集请点赞关注收藏后评论区留言私信~~~

    一、视觉问题简介

    视觉问答（VQA）是一种同时设计计算机视觉和自然语言处理的学习任务。简单来说，VQA就是对给定的图片进行问答，一个VQA系统以一张图片和一个关于这张图片形式自由，开放式的自然语言问题作为输入，生成一条自然语言答案作为输出，视觉问题系统综合运用到了目前的计算机视觉和自然语言处理的技术，并设计模型设计，实验，以及可视化。

    VQA问题的一种典型模型是联合嵌入模型，这种方法首先学习视觉与自然语言的两个不同模态特征在一个共同的特征空间的嵌入表示，然后根据这种嵌入表示产生回答。

    二、数据集的准备

    1：下载数据

    这里使用VQA2.0数据集进行训练和验证，VQA2.0是一个公认有难度，并且语言验证得到了有效控制的数据集

    本次使用到的图片为MSCOCO数据集中train2014子集和val2014子集，图片可以在官网下载

    数据集网址

    本次用到的图像特征是由目标检测网络Faster-RCNN检测并生成的，可评论区留言私信博主要

    2：安装依赖

    确保安装好PyTorch，然后在程序目录下运行pip install -r requirements.txt安装其他依赖项

    三、关键模块简介

    1：FCnet模块

    FCnet即一系列的全连接层，各个层的输入输出大小在模块构建时给出，这个模块默认使其中的全连接层具有bias，并以ReLU作为激活函数 并使用weight normalization

    2：SimpleClassifier模块

    它的作用是：在视觉问答系统的末端，根据融合的特征得到最终答案

    3：问题嵌入模块

    在联合嵌入模型中，需要使用RNN将输入的问题编码成向量，LSTM和GRU使两种代表性的RNN，由于实践中GRU与LSTM表现相近且占用显存较少，所以这里选用GRU

    4：词嵌入

    要获得问题句子的嵌入表示，首先应该获得词嵌入表示，每一个词需要用一个唯一的数字表示

    baseline代码如下

    1. import torch
    2. import torch.nn as nn
    3. from lib.module import topdown_attention
    4. from lib.module.language_model import WordEmbedding, QuestionEmbedding
    5. from lib.module.classifier import SimpleClassifier
    6. from lib.module.fc import FCNet
    7.  
    8. class Baseline(nn.Module):
    9.  
    10.     def __init__(self, w_emb, q_emb, v_att, q_net, v_net, classifer, need_internals=False):
    11.         super(Baseline, self).__init__()
    12.  
    13.         self.need_internals = need_internals
    14.  
    15.         self.w_emb = w_emb
    16.         self.q_emb = q_emb
    17.         self.v_att = v_att
    18.         self.q_net = q_net
    19.         self.v_net = v_net
    20.         self.classifier = classifer
    21.  
    22.     def forward(self, q_tokens, ent_features):
    23.  
    24.         w_emb = self.w_emb(q_tokens)
    25.         q_emb = self.q_emb(w_emb)
    26.  
    27.         att = self.v_att(q_emb, ent_features) # [ B, n_ent, 1 ]
    28.         v_emb = (att * ent_features).sum(1)  # [ B, hid_dim ]
    29.  
    30.         internals = [att.squeeze()] if self.need_internals else None
    31.  
    32.         q_repr = self.q_net(q_emb)
    33.         v_repr = self.v_net(v_emb)
    34.         joint_repr = q_repr * v_repr
    35.         logits = self.classifier(joint_repr)
    36.  
    37.         return logits, internals
    38.  
    39.     @classmethod
    40.     def build_from_config(cls, cfg, dataset, need_internals):
    41.         w_emb = WordEmbedding(dataset.word_dict.n_tokens, cfg.lm.word_emb_dim, 0.0)
    42.         q_emb = QuestionEmbedding(cfg.lm.word_emb_dim, cfg.hid_dim, cfg.lm.n_layers, cfg.lm.bidirectional, cfg.lm.dropout, cfg.lm.rnn_type)
    43.         q_dim = cfg.hid_dim
    44.         att_cls = topdown_attention.classes[cfg.topdown_att.type]
    45.         v_att = att_cls(1, q_dim, cfg.ent_dim, cfg.topdown_att.hid_dim, cfg.topdown_att.dropout)
    46.         q_net = FCNet([q_dim, cfg.hid_dim])
    47.         v_net = FCNet([cfg.ent_dim, cfg.hid_dim])
    48.         classifier = SimpleClassifier(cfg.hid_dim, cfg.mlp.hid_dim, dataset.ans_dict.n_tokens, cfg.mlp.dropout)
    49.         return cls(w_emb, q_emb, v_att, q_net, v_net, classifier, need_internals)

    数据集目录如下

    四、结果可视化

    读取了之前训练好的模型之后，使用数据为配置文件中的val，程序运行完成后结果可视化如下

    机器对于给出的图片会输出对于的问答结果

     

     五、代码

    部分代码如下

    训练类

    1. import os
    2. import time
    3. import torch
    4. import torch.nn as nn
    5. import torch.nn.functional as F
    6. from torch.optim.lr_scheduler import LambdaLR
    7. from torch.nn.utils import clip_grad_norm_
    8. from bisect import bisect
    9. from tqdm import tqdm
    10.  
    11. def bce_with_logits(logits, labels):
    12.     assert logits.dim() == 2
    13.     loss = F.binary_cross_entropy_with_logits(logits, labels)
    14.     loss *= labels.size(1) # multiply by number of QAs
    15.     return loss
    16.  
    17. def sce_with_logits(logits, labels):
    18.     assert logits.dim() == 2
    19.     loss = F.cross_entropy(logits, labels.nonzero()[:, 1])
    20.     loss *= labels.size(1)
    21.     return loss
    22.  
    23. def compute_score_with_logits(logits, labels):
    24.     with torch.no_grad():
    25.         logits = torch.max(logits, 1)[1] # argmax
    26.         one_hots = torch.zeros(*labels.size()).cuda()
    27.         one_hots.scatter_(1, logits.view(-1, 1), 1)
    28.         scores = (one_hots * labels)
    29.         return scores
    30.  
    31. def lr_schedule_func_builder(cfg):
    32.     def func(step_idx):
    33.         if step_idx <= cfg.train.warmup_steps:
    34.             alpha = float(step_idx) / float(cfg.train.warmup_steps)
    35.             return cfg.train.warmup_factor * (1. - alpha) + alpha
    36.         else:
    37.             idx = bisect(cfg.train.lr_steps, step_idx)
    38.             return pow(cfg.train.lr_ratio, idx)
    39.     return func
    40.  
    41. def train(model, cfg, train_loader, val_loader, n_epochs, val_freq, out_dir):
    42.  
    43.     os.makedirs(out_dir, exist_ok=True)
    44.     optim = torch.optim.Adamax(model.parameters(), **cfg.train.optim)
    45.  
    46.     n_train_batches = len(train_loader)
    47.  
    48.     train_score = 0.0
    49.     loss_fn = bce_with_logits if cfg.model.loss == "logistic" else sce_with_logits
    50.  
    51.     for epoch in range(n_epochs):
    52.  
    53.         epoch_loss = 0.0
    54.  
    55.         tic_0 = time.time()
    56.  
    57.         for i, data in enumerate(train_loader):
    58.  
    59.             tic_1 = time.time()
    60.  
    61.             q_tokens = data[2].cuda()
    62.             a_targets = data[3].cuda()
    63.             v_features = [_.cuda() for _ in data[4:]]
    64.  
    65.             tic_2 = time.time()
    66.  
    67.             optim.zero_grad()
    68.             logits, _ = model(q_tokens, *v_features)
    69.             loss = loss_fn(logits, a_targets)
    70.  
    71.             tic_3 = time.time()
    72.  
    73.             loss.backward()
    74.             if cfg.train.clip_grad: clip_grad_norm_(model.parameters(), cfg.train.max_grad_norm)
    75.             optim.step()
    76.  
    77.             tic_4 = time.time()
    78.  
    79.             batch_score = compute_score_with_logits(logits, a_targets).sum()
    80.             epoch_loss += float(loss.data.item() * logits.size(0))
    81.             train_score += float(batch_score)
    82.  
    83.             del loss
    84.  
    85.             logstr = "epoch %2d batch %4d/%4d | ^ %4dms | => %4dms | <= %4dms" % \
    86.                   (epoch + 1, i + 1, n_train_batches, 1000*(tic_2-tic_0), 1000*(tic_3-tic_2), 1000*(tic_4-tic_3))
    87.             print("%-80s" % logstr, end="\r")
    88.  
    89.             tic_0 = time.time()
    90.  
    91.         epoch_loss /= len(train_loader.dataset)
    92.         train_score = 100 * train_score / len(train_loader.dataset)
    93.  
    94.         logstr = "epoch %2d | train_loss: %5.2f train_score: %5.2f" % (epoch + 1, epoch_loss, train_score)
    95.         if (epoch + 1) % val_freq == 0:
    96.             model.eval()
    97.             val_score, upper_bound = validate(model, val_loader)
    98.             model.train()
    99.             logstr += " | val_score: %5.2f (%5.2f)" % (100 * val_score, 100 * upper_bound)
    100.         print("%-80s" % logstr)
    101.  
    102.         model_path = os.path.join(out_dir, 'model_%d.pth' % (epoch + 1))
    103.         torch.save(model.state_dict(), model_path)
    104.  
    105. def validate(model, loader):
    106.  
    107.     score = 0
    108.     upper_bound = 0
    109.     n_qas = 0
    110.  
    111.     with torch.no_grad():
    112.  
    113.         for i, data in enumerate(loader):
    114.  
    115.             q_tokens = data[2].cuda()
    116.             a_targets = data[3].cuda()
    117.             v_features = [_.cuda() for _ in data[4:]]
    118.  
    119.             logits, _ = model(q_tokens, *v_features)
    120.             batch_score = compute_score_with_logits(logits, a_targets)
    121.             score += batch_score.sum()
    122.  
    123.             upper_bound += (a_targets.max(1)[0]).sum()
    124.             n_qas += logits.size(0)
    125.  
    126.             logstr = "val batch %5d/%5d" % (i + 1, len(loader))
    127.             print("%-80s" % logstr, end='\r')
    128.  
    129.     score = score / n_qas
    130.     upper_bound = upper_bound / n_qas
    131.     return score, upper_bound

    infer类

    1. import os
    2. import time
    3. import json
    4. import torch
    5. import cv2
    6. import shutil
    7. import numpy as np
    8. import torch.nn as nn
    9. import torch.nn.functional as F
    10. from tqdm import tqdm
    11.  
    12. colors = [ (175, 84, 65), (68, 194, 246), (136, 147, 65), (92, 192, 151) ]
    13.  
    14. def attention_map(im, boxes, atts, p=0.8, bgc=1.0, compress=0.85, box_color=(65, 81, 226)):
    15.  
    16.     height, width, channel = im.shape
    17.     im = im / 255.0
    18.  
    19.     att_map = np.zeros([height, width])
    20.     boxes = boxes.astype(np.int)
    21.     for box, att in zip(boxes, atts):
    22.         x1, y1, x2, y2 = box
    23.         roi = att_map[y1:y2, x1:x2]
    24.         roi[roi < att] = att
    25.  
    26.     att_map /= att_map.max()
    27.     att_map = att_map ** p
    28.     att_map = att_map * compress + (1-compress)
    29.  
    30.     att_map = cv2.resize(att_map, (int(width/16), int(height/16)))
    31.     att_map = cv2.resize(att_map, (width, height))
    32.  
    33.     att_map = np.expand_dims(att_map, axis=2)
    34.  
    35.     bg = np.ones_like(att_map) * bgc
    36.     att_im = im * att_map + bg * (1-att_map)
    37.  
    38.     att_im = (att_im * 255).astype(np.uint8)
    39.  
    40.     center = np.argmax(atts)
    41.     x1, y1, x2, y2 = boxes[center]
    42.     cv2.rectangle(att_im, (x1, y1), (x2, y2), box_color, 5)
    43.  
    44.     return att_im
    45.  
    46. def infer_visualize(model, args, cfg, ans_dict, loader):
    47.  
    48.     _, ckpt = os.path.split(args.checkpoint)
    49.     ckpt, _ = os.path.splitext(ckpt)
    50.     out_dir = os.path.join(args.out_dir, "%s_%s_%s_visualization" % (args.cfg_name, ckpt, args.data))
    51.     os.makedirs(out_dir, exist_ok=True)
    52.     model.eval()
    53.  
    54.     questions_path = cfg.data[args.data].composition[0].q_jsons[0]
    55.     questions = json.load(open(questions_path))
    56.  
    57.     pbar = tqdm(total=args.n_batches * loader.batch_size)
    58.  
    59.     with torch.no_grad():
    60.  
    61.         for i, data in enumerate(loader):
    62.  
    63.             if i == args.n_batches: break
    64.  
    65.             question_ids = data[0]
    66.             image_ids = data[1]
    67.             q_tokens = data[2].cuda()
    68.             obj_featuers = data[4].cuda()
    69.             batch_boxes = data[5].numpy()
    70.  
    71.             logits, internals = model(q_tokens, obj_featuers)
    72.             topdown_atts = internals[0]
    73.             topdown_atts = topdown_atts.data.cpu().numpy()
    74.             _, predictions = logits.max(dim=1)
    75.  
    76.             for idx in range(len(question_ids)):
    77.  
    78.                 question_id = question_ids[idx]
    79.                 image_id = image_ids[idx]
    80.                 boxes = batch_boxes[idx]
    81.                 answer = ans_dict.idx2ans[predictions[idx]]
    82.                 q_entry = questions[question_id]
    83.                 topdown_att = topdown_atts[idx]
    84.                 question = q_entry["question"]
    85.                 gts = list(q_entry["answers"].items())
    86.                 gts = sorted(gts, reverse=True, key=lambda x: x[1])
    87.                 gt = gts[0][0]
    88.  
    89.                 q_out_dir = os.path.join(out_dir, question_id)
    90.                 os.makedirs(q_out_dir, exist_ok=True)
    91.  
    92.                 q_str = question + "\n" + "gt: %s\n" % gt + "answer: %s\n" % answer
    93.                 with open(os.path.join(q_out_dir, "qa.txt"), "w") as f: f.write(q_str)
    94.  
    95.                 image_path = os.path.join(args.images_dir, "%s.jpg" % image_id)
    96.                 shutil.copy(image_path, os.path.join(q_out_dir, "original.jpg"))
    97.  
    98.                 im = cv2.imread(image_path)
    99.                 att_map = attention_map(im.copy(), boxes, topdown_att)
    100.                 cv2.imwrite(os.path.join(q_out_dir, "topdown_att.jpg"), att_map)
    101.  
    102.                 pbar.update(1)

    创作不易 觉得有帮助请点赞关注收藏~~~

  • 相关阅读:
    面试题-01
    工控机通过Profinet转Modbus RTU网关连接变频器与电机通讯案例
    .NET桌面程序集成Web网页开发的十种解决方案
    软件测试工程师需要掌握哪些技能呢？
    逻辑扇区和物理扇区
    代码随想录算法训练营第四十五天丨 动态规划part08
    2023年【广东省安全员C证第四批（专职安全生产管理人员）】报名考试及广东省安全员C证第四批（专职安全生产管理人员）最新解析
    鲲鹏代码迁移工具介绍
    const对象竟然可以修改其成员，只因顶层const
    【Glide 】框架在内存使用方面的优化
  • 原文地址：https://blog.csdn.net/jiebaoshayebuhui/article/details/128024996