• PyTorch搭建卷积神经网络（CNN）进行视频行为识别（附源码和数据集）

    需要数据集和源码请点赞关注收藏后评论区留下QQ邮箱~~~

    一、行为识别简介

    行为识别是视频理解中的一项基础任务，它可以从视频中提取语义信息，进而可以为其他任务如行为检测，行为定位等提供通用的视频表征

    现有的视频行为数据集大致可以划分为两种类型

    1：场景相关数据集  这一类的数据集场景提供了较多的语义信息 仅仅通过单帧图像便能很好的判断对应的行为 

    2：时序相关数据集  这一类数据集对时间关系要求很高，需要足够多帧图像才能准确的识别视频中的行为。

    例如骑马的例子就与场景高度相关，马和草地给出了足够多的语义信息

    但是打开柜子就与时间高度相关，如果反转时序甚至容易认为在关闭柜子

     如下图

     

     二、数据准备

    数据的准备包括对视频的抽帧处理，具体原理此处不再赘述

    大家可自行前往官网下载数据集

    视频行为识别数据集

    三、模型搭建与训练

    在介绍模型的搭建与训练之外，需要先了解的命令行参数，还有无名的必填参数dataset以及modality。前者用于选择数据集，后者用于确定数据集类型 是RGB图像还是Flow光流图像

    过程比较繁琐 此处不再赘述

    效果如下图

    最终会得到如下的热力图，从红色到黄色到绿色到蓝色，网络的关注度从大到小，可以看到模块可以很好地定位到运动发生的时空区域 

    四、代码 

    项目结构如下

     

    main函数代码

    1.  
    2. import os
    3. import time
    4. import shutil
    5. import torch.nn.parallel
    6. imd_norm_
    7.  
    8. from ops.dataset import TSNDataSet
    9. from ops.models import TSN
    10. from ops.transforms import *
    11. from opts import parser
    12. from ops import dataset_config
    13. from ops.utils import AverageMeter, accuracy
    14. from ops.temporal_shift import make_temporal_pool
    15.  
    16. from tensorboardX import SummaryWriter
    17.  
    18. best_prec1 = 0
    19.  
    20.  
    21. def main():
    22.     global args, best_prec1
    23.     args = parser.parse_args()
    24.  
    25.     num_class, args.train_list, args.val_list, args.root_path, prefix = dataset_config.return_dataset(args.dataset,
    26.                                                                                                       args.modality)
    27.     full_arch_name = args.arch
    28.     if args.shift:
    29.         full_arch_name += '_shift{}_{}'.format(args.shift_div, args.shift_place)
    30.     if args.temporal_pool:
    31.         full_arch_name += '_tpool'
    32.     args.store_name = '_'.join(
    33.         ['TSM', args.dataset, args.modality, full_arch_name, args.consensus_type, 'segment%d' % args.num_segments,
    34.          'e{}'.format(args.epochs)])
    35.  
    36.         args.store_name += '_nl'
    37.     if args.suffix is not None:
    38.         args.store_name += '_{}'.format(args.suffix)
    39.     print('storing name: ' + args.store_name)
    40.  
    41.     check_rootfolders()
    42.  
    43.     model = TSN(num_class, args.num_segments, args.modality,
    44.                 base_model=args.arch,
    45.                 consensus_type=args.consensus_type,
    46.                 dropout=args.dropout,
    47.                 img_feature_dim=args.img_feature_dim,
    48.                 partial_bn=not args.no_partialbn,
    49.                 pretrain=args.pretrain,
    50.                 is_shift=args.shift, shift_div=args.shift_div, shift_place=args.shift_place,
    51.                 fc_lr5=not (args.tune_from and args.dataset in args.tune_from),
    52.                 temporal_pool=args.temporal_pool,
    53.                 non_local=args.non_local)
    54.  
    55.     crop_size = model.crop_size
    56.     scale_size = model.scale_size
    57.     input_mean = model.input_mean
    58.     in else True)
    59.  
    60.     model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
    61.  
    62.     optimizer = torch.optim.SGD(policies,
    63.                                 args.lr,
    64.                                 momentum=args.momentum,
    65.                                 weight_decay=args.weight_decay)
    66.  
    67.     if args.resume:
    68.         if args.temporal_pool:  # early temporal pool so that we can load the state_dict
    69.             make_temporal_pool(model.module.base_model, args.num_segments)
    70.         if os.path.isfile(args.resume):
    71.             print(("=> loading checkpoint '{}'".format(args.resume)))
    72.             checkpoint = torch.load(args.resume)
    73.             args.start_epoch = checkpoint['epoch']
    74.             best_prec1 = checkpoint['best_prec1']
    75.             model.load_state_dict(checkpoint['state_dict'])
    76.             optimizer.load_state_dict(checkpoint['optimizer'])
    77.             print(("=> loaded checkpoint '{}' (epoch {})"
    78.                    .format(args.evaluate, checkpoint['epoch'])))
    79.         else:
    80.             print(("=> no checkpoint found at '{}'".format(args.resume)))
    81.  
    82.   ate_dict']
    83.         model_dict = model.state_dict()
    84.         replace_dict = []
    85.         for k, v in sd.items():
    86.             if k not in model_dict and k.replace('.net', '') in model_dict:
    87.                 print('=> Load after remove .net: ', k)
    88.                 replace_dict.append((k, k.replace('.net', '')))
    89.         for k, v in model_dict.items():
    90.             if k not in sd and k.replace('.net', '') in sd:
    91.                 print('=> Load after adding .net: ', k)
    92.                 replace_dict.append((k.replace('.net', ''), k))
    94.         for k, k_new in replace_dict:
    95.             sd[k_new] = sd.pop(k)
    96.         keys1 = set(list(sd.keys()))
    97.         keys2 = set(list(model_dict.keys()))
    98.         set_diff = (keys1 - keys2) | (keys2 - keys1)
    99.         print('#### Notice: keys that failed to load: {}'.format(set_diff))
    100.         if args.dataset not in args.tune_from:  # new dataset
    101.             print('=> New dataset, do not load fc weights')
    102.             sd = {k: v for k, v in sd.items() if 'fc' not in k}
    103.         if te_dict(model_dict)
    105.     if args.temporal_pool and not args.resume:
    106.         make_temporal_pool(model.module.base_model, args.num_segments)
    108.     cudnn.benchmark = True
    110.     # Data loading code
    111.     if args.modality != 'RGBDiff':
    112.         normalize = GroupNormalize(input_mean, input_std)
    113.     else:
    114.         normalize = IdentityTransform()
    116.     if args.modality == 'RGB':
    117.         data_length = 1
    118.     elif args.modality in ['Flow', 'RGBDiff']:
    119.         data_length = 5
    121.     train_loader = torch.utils.data.DataLoader(
    122.         TSNDataSet(args.root_path, args.train_list, num_segments=args.num_segments,
    123.                    new_length=data_length,
    124.                    modality=args.modality,
    125.                    image_tmpl=prefix,
    126.                    transform=torchvision.transforms.Compose([
    127.                        train_augmentation,
    128.                        Stack(roll=(args.arch in ['BNInception', 'InceptionV3'])),
    129.                        ToTorchFormatTensor(div=(args.arch not in ['BNInception', 'InceptionV3'])),
    130.                        normalize,
    131.                    ]), dense_sample=args.dense_sample),
    132.         batch_size=args.batch_size, shuffle=True,
    133.         num_workers=args.workers, pin_memory=True,
    134.         drop_last=True)  # prevent something not % n_GPU
    136.     val_loader = torch.utils.data.DataLoader(
    137.         TSNDataSet(args.root_path, args.val_list, num_segments=args.num_segments,
    138.                    new_length=data_length,
    139.                    modality=args.modality,
    140.                    image_tmpl=prefix,
    141.                    random_shift=False,
    142.                    transform=torchvision.transforms.Compose([
    143.                        GroupScale(int(scale_size)),
    144.                        GroupCenterCrop(crop_size),
    145.                        Stack(roll=(args.arch in ['BNInception', 'InceptionV3'])),
    146.                        ToTorchFormatTensor(div=(args.arch not in ['BNInception', 'InceptionV3'])),
    147.                        normalize,
    148.                    ]), dense_sample=args.dense_sample),
    149.         batch_size=args.batch_size, shuffle=False,
    150.         num_workers=args.workers, pin_memory=True)
    152.     # define loss function (criterion) and optimizer
    153.     if args.loss_type == 'nll':
    154.         criterion = torch.nn.CrossEntropyLoss().cuda()
    155.     else:
    156.         raise ValueError("Unknown loss type")
    158.     for group in policies:
    159.         print(('group: {} has {} params, lr_mult: {}, decay_mult: {}'.format(
    160.             group['name'], len(group['params']), group['lr_mult'], group['decay_mult'])))
    162.     if args.evaluate:
    163.         validate(val_loader, model, criterion, 0)
    164.         return
    166.     log_training = open(os.path.join(args.root_log, args.store_name, 'log.csv'), 'w')
    167.     with open(os.path.join(args.root_log, args.store_name, 'args.txt'), 'w') as f:
    168.         f.write(str(args))
    169.     tf_writer = SummaryWriter(log_dir=os.path.join(args.root_log, args.store_name))
    170.     for epoch in range(args.start_epoch, args.epochs):
    171.         adjust_learning_rate(optimizer, epoch, args.lr_type, args.lr_steps)
    173.         # train for one epoch
    174.         train(train_loader, model, criterion, optimizer, epoch, log_training, tf_writer)
    176.         # evaluate on validation set
    177.         if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1:
    178.             prec1 = validate(val_loader, model, criterion, epoch, log_training, tf_writer)
    180.             # remember best prec@1 and save checkpoint
    181.             is_best = prec1 > best_prec1
    182.             best_prec1 = max(prec1, best_prec1)
    183.             tf_writer.add_scalar('acc/test_top1_best', best_prec1, epoch)
    185.             output_best = 'Best Prec@1: %.3f\n' % (best_prec1)
    186.             print(output_best)
    187.             log_training.write(output_best + '\n')
    188.             log_training.flush()
    190.             save_checkpoint({
    191.                 'epoch': epoch + 1,
    192.                 'arch': args.arch,
    193.                 'state_dict': model.state_dict(),
    194.                 'optimizer': optimizer.state_dict(),
    195.                 'best_prec1': best_prec1,
    196.             }, is_best)
    199. def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer):
    200.     batch_time = AverageMeter()
    201.     data_time = AverageMeter()
    202.     losses = AverageMeter()
    203.     top1 = AverageMeter()
    204.     top5 = AverageMeter()
    206.     if args.no_partialbn:
    207.         model.module.partialBN(False)
    208.     else:
    209.         model.module.partialBN(True)
    211.     # switch to train mode
    212.     model.train()
    214.     end = time.time()
    215.     for i, (input, target) in enumerate(train_loader):
    216.         # measure data loading time
    217.         data_time.update(time.time() - end)
    219.         target = target.cuda()
    220.         input_var = torch.autograd.Variable(input)
    221.         target_var = torch.autograd.Variable(target)
    223.         # compute output
    224.         output = model(input_var)
    225.         loss = criterion(output, target_var)
    227.         # measure accuracy and record loss
    228.         prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
    229.         losses.update(loss.item(), input.size(0))
    230.         top1.update(prec1.item(), input.size(0))
    231.         top5.update(prec5.item(), input.size(0))
    233.         # compute gradient and do SGD step
    234.         loss.backward()
    236.         if args.clip_gradient is not None:
    237.             total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient)
    239.         optimizer.step()
    240.         optimizer.zero_grad()
    242.         # measure elapsed time
    243.         batch_time.update(time.time() - end)
    244.         end = time.time()
    246.         if i % args.print_freq == 0:
    247.             output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
    248.                       'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
    249.                       'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
    250.                       'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
    251.                       'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
    252.                       'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
    253.                 epoch, i, len(train_loader), batch_time=batch_time,
    254.                 data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1))  # TODO
    255.             print(output)
    256.             log.write(output + '\n')
    257.             log.flush()
    259.     tf_writer.add_scalar('loss/train', losses.avg, epoch)
    260.     tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
    261.     tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
    262.     tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
    265. def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None):
    266.     batch_time = AverageMeter()
    267.     losses = AverageMeter()
    268.     top1 = AverageMeter()
    269.     top5 = AverageMeter()
    271.     # switch to evaluate mode
    272.     model.eval()
    274.     end = time.time()
    275.     with torch.no_grad():
    276.         for i, (input, target) in enumerate(val_loader):
    277.             target = target.cuda()
    279.             # compute output
    280.             output = model(input)
    281.             loss = criterion(output, target)
    283.             # measure accuracy and record loss
    284.             prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
    286.             losses.update(loss.item(), input.size(0))
    287.             top1.update(prec1.item(), input.size(0))
    288.             top5.update(prec5.item(), input.size(0))
    290.             # measure elapsed time
    291.             batch_time.update(time.time() - end)
    292.             end = time.time()
    294.             if i % args.print_freq == 0:
    295.                 output = ('Test: [{0}/{1}]\t'
    296.                           'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
    297.                           'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
    298.                           'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
    299.                           'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
    300.                     i, len(val_loader), batch_time=batch_time, loss=losses,
    301.                     top1=top1, top5=top5))
    302.                 print(output)
    303.                 if log is not None:
    304.                     log.write(output + '\n')
    305.                     log.flush()
    307.     output = ('Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
    308.               .format(top1=top1, top5=top5, loss=losses))
    309.     print(output)
    310.     if log is not None:
    311.         log.write(output + '\n')
    312.         log.flush()
    314.     if tf_writer is not None:
    315.         tf_writer.add_scalar('loss/test', losses.avg, epoch)
    316.         tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
    317.         tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
    319.     return top1.avg
    322. def save_checkpoint(state, is_best):
    323.     filename = '%s/%s/ckpt.pth.tar' % (args.root_model, args.store_name)
    324.     torch.save(state, filename)
    325.     if is_best:
    326.         shutil.copyfile(filename, filename.replace('pth.tar', 'best.pth.tar'))
    329. def adjust_learning_rate(optimizer, epoch, lr_type, lr_steps):
    330.     """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    331.     if lr_type == 'step':
    332.         decay = 0.1 ** (sum(epoch >= np.array(lr_steps)))
    333.         lr = args.lr * decay
    334.         decay = args.weight_decay
    335.     elif lr_type == 'cos':
    336.         import math
    337.         lr = 0.5 * args.lr * (1 + math.cos(math.pi * epoch / args.epochs))
    338.         decay = args.weight_decay
    339.     else:
    340.         raise NotImplementedError
    341.     for param_group in optimizer.param_groups:
    342.         param_group['lr'] = lr * param_group['lr_mult']
    343.         param_group['weight_decay'] = decay * param_group['decay_mult']
    346. def check_rootfolders():
    347.     """Create log and model folder"""
    348.     folders_util = [args.root_log, args.root_model,
    349.                     os.path.join(args.root_log, args.store_name),
    350.                     os.path.join(args.root_model, args.store_name)]
    351.     for folder in folders_util:
    352.         if not os.path.exists(folder):
    353.             print('creating folder ' + folder)
    354.             os.mkdir(folder)
    357. if __name__ == '__main__':
    358.     main()

    opts类代码如下

    1. #这里下面的参数应该要自行输入
    2.  
    3.  
    4. import argparse
    5. parser = argparse.ArgumentParser(description="PyTorch implementation of Temporal Segment Networks")
    6. parser.add_argument('dataset', default="")
    7. parser.add_argument('modality', default="RGB", choices=['RGB', 'Flow'])
    8. parser.add_argument('--train_list', type=str, default="")
    9. parser.add_argument('--val_list', type=str, default="")
    10. parser.add_argument('--root_path', type=str, default="")
    11. parser.add_argument('--store_name', type=str, default="")
    12. # ========================= Model Configs ==========================
    13. parser.add_argument('--arch', type=str, default="BNInception")
    14. parser.add_argument('--num_segments', type=int, default=3)
    15. parser.add_argument('--consensus_type', type=str, default='avg')
    16. parser.add_argument('--k', type=int, default=3)
    17.  
    18. parser.add_argument('--dropout', '--do', default=0.5, type=float,
    19.                     metavar='DO', help='dropout ratio (default: 0.5)')
    20. parser.add_argument('--loss_type', type=str, default="nll",
    21.                     choices=['nll'])
    22. parser.add_argument('--img_feature_dim', default=256, type=int, help="the feature dimension for each frame")
    23. parser.add_argument('--suffix', type=str, default=None)
    24. parser.add_argument('--pretrain', type=str, default='imagenet')
    25. parser.add_argument('--tune_from', type=str, default=None, help='fine-tune from checkpoint')
    26.  
    27. # ========================= Learning Configs ==========================
    28. parser.add_argument('--epochs', default=120, type=int, metavar='N',
    29.                     help='number of total epochs to run')
    30. parser.add_argument('-b', '--batch-size', default=128, type=int,
    31.                     metavar='N', help='mini-batch size (default: 256)')
    32. parser.add_argument('--lr', '--learning-rate', default=0.001, type=float,
    33.                     metavar='LR', help='initial learning rate')
    34. parser.add_argument('--lr_type', default='step', type=str,
    35.                     metavar='LRtype', help='learning rate type')
    36. parser.add_argument('--lr_steps', default=[50, 100], type=float, nargs="+",
    37.                     metavar='LRSteps', help='epochs to decay learning rate by 10')
    38. parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
    39.                     help='momentum')
    40. parser.add_argument('--weight-decay', '--wd', default=5e-4, type=float,
    41.                     metavar='W', help='weight decay (default: 5e-4)')
    42. parser.add_argument('--clip-gradient', '--gd', default=None, type=float,
    43.                     metavar='W', help='gradient norm clipping (default: disabled)')
    44. parser.add_argument('--no_partialbn', '--npb', default=False, action="store_true")
    45.  
    46. # ========================= Monitor Configs ==========================
    47. parser.add_argument('--print-freq', '-p', default=20, type=int,
    48.                     metavar='N', help='print frequency (default: 10)')
    49. parser.add_argument('--eval-freq', '-ef', default=5, type=int,
    50.                     metavar='N', help='evaluation frequency (default: 5)')
    51.  
    52.  
    53. # ========================= Runtime Configs ==========================
    54. parser.add_argument('-j', '--workers', default=8, type=int, metavar='N',
    55.                     help='number of data loading workers (default: 8)')
    56. parser.add_argument('--resume', default='', type=str, metavar='PATH',
    57.                     help='path to latest checkpoint (default: none)')
    58. parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
    59.                     help='evaluate model on validation set')
    60. parser.add_argument('--snapshot_pref', type=str, default="")
    61. parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
    62.                     help='manual epoch number (useful on restarts)')
    63. parser.add_argument('--gpus', nargs='+', type=int, default=None)
    64. parser.add_argument('--flow_prefix', default="", type=str)
    65. parser.add_argument('--root_log',type=str, default='log')
    66. parser.add_argument('--root_model', type=str, default='checkpoint')
    67.  
    68. parser.add_argument('--shift', default=False, action="store_true", help='use shift for models')
    69. parser.add_argument('--shift_div', default=8, type=int, help='number of div for shift (default: 8)')
    70. parser.add_argument('--shift_place', default='blockres', type=str, help='place for shift (default: stageres)')
    71.  
    72. parser.add_argument('--temporal_pool', default=False, action="store_true", help='add temporal pooling')
    73. parser.add_argument('--non_local', default=False, action="store_true", help='add non local block')
    74.  
    75. parser.add_argument('--dense_sample', default=False, action="store_true", help='use dense sample for video dataset')

    test_models类代码如下

    1.  
    2.  
    3. # Notice that this file has been modified to support ensemble testing
    4.  
    5.  
    6. from ops.transforms import *
    7. from ops import dataset_config
    8. from torch.nn import functional as F
    9.  
    10. # options
    11. parser = argparse.ArgumentParser(description="TSM testing on the full validation set")
    12. parser.add_argument('dataset', type=str)
    13.  
    14. # may contain splits
    15. pars
    16. parser.add_argument('--test_crops', type=int, default=1)
    17. parser.add_argument('--coeff', type=str, default=None)
    18. parser.add_argument('--batch_size', type=int, default=1)
    19. parser.add_argument('-j', '--workers', default=8, type=int, metavar='N',
    20.                     help='number of data loading workers (default: 8)')
    21.  
    22. # for true test
    23. parser.add_argument('--test_list', type=str, default=None)
    24. parser.add_argument('--csv_file', type=str, default=None)
    25.  
    26. parser.add_argument('--softmax', default=False, action="store_true", help='use softmax')
    27.  
    28. parser.add_argument('--max_num', type=int, default=-1)
    29. parser.add_argument('--input_size', type=int, default=224)
    30. parser.add_argument('--crop_fusion_type', type=str, default='avg')
    31. parser.add_argument('--gpus', nargs='+', type=int, default=None)
    32. parser.add_argument('--img_feature_dim',type=int, default=256)
    33. parser.add_argument('--num_set_segments',type=int, default=1,help='TODO: select multiply set of n-frames from a video')
    34. parser.add_argument('--pretrain', type=str, default='imagenet')
    35.  
    36. args = parser.parse_args()
    37.  
    38.  
    39. class AverageMeter(object):
    40.     """Computes and stores the average and current value"""
    41.     def __init__(self):
    42.         self.reset()
    43.  
    44.     def reset(self):
    45.         self.val = 0
    46.         self.avg = 0
    47.         self.sum = 0
    48.         self.count = 0
    49.  
    50.     def update(self, val, n=1):
    51.         self.val = val
    52.         self.sum += val * n
    53.         self.count += n
    54.         self.avg = self.sum / self.count
    55.  
    56.  
    57. def accuracy(output, target, topk=(1,)):
    58.     """Computes the precision@k for the specified values of k"""
    59.     maxk = max(topk)
    60.     batch_size = target.size(0)
    61.     _, pred = output.topk(maxk, 1, True, True)
    62.     pred = pred.t()
    63.     correct = pred.eq(target.view(1, -1).expand_as(pred))
    64.     res = []
    65.     for k in topk:
    66.          correct_k = correct[:k].view(-1).float().sum(0)
    67.          res.append(correct_k.mul_(100.0 / batch_size))
    68.     return res
    69.  
    70.  
    71. def parse_shift_option_from_log_name(log_name):
    72.     if 'shift' in log_name:
    73.         strings = log_name.split('_')
    74.         for i, s in enumerate(strings):
    75.             if 'shift' in s:
    76.                 break
    77.         return True, int(strings[i].replace('shift', '')), strings[i + 1]
    78.     else:
    79.         return False, None, None
    80.  
    81.  
    82. weights_list = args.weights.split(',')
    83. test_segments_list = [int(s) for s in args.test_segments.split(',')]
    84. assert len(weights_list) == len(test_segments_list)
    85. if args.coeff is None:
    86.     coeff_list = [1] * len(weights_list)
    87. else:
    88.     coeff_list = [float(c) for c in args.coeff.split(',')]
    89.  
    90. if args.test_list is not None:
    91.     test_file_list = args.test_list.split(',')
    92. else:
    93.     test_file_list = [None] * len(weights_list)
    94.  
    95.  
    96. data_iter_list = []
    97. net_list = []
    98. modality_list = []
    99.  
    100. total_num = None
    101. for this_weights, this_test_segments, test_file in zip(weights_list, test_segments_list, test_file_list):
    102.     is_shift, shift_div, shift_place = parse_shift_option_from_log_name(this_weights)
    103.     if 'RGB' in this_weights:
    104.         modality = 'RGB'
    105.     else:
    106.         modality = 'Flow'
    107.     this_arch = this_weights.split('TSM_')[1].split('_')[2]
    108.     modality_list.append(modality)
    109.     num_class, args.train_list, val_list, root_path, prefix = dataset_config.return_dataset(args.dataset,
    110.                                                                                             modality)
    111.     print('=> shift: {}, shift_div: {}, shift_place: {}'.format(is_shift, shift_div, shift_place))
    112.     net = TSN(num_class, this_test_segments if is_shift else 1, modality,
    113.               base_model=this_arch,
    114.               consensus_type=args.crop_fusion_type,
    115.               img_feature_dim=args.img_feature_dim,
    116.               pretrain=args.pretrain,
    117.               is_shift=is_shift, shift_div=shift_div, shift_place=shift_place,
    118.               non_local='_nl' in this_weights,
    119.               )
    120.  
    121.     if 'tpool' in this_weights:
    122.         from ops.temporal_shift import make_temporal_pool
    123.         make_temporal_pool(net.base_model, this_test_segments)  # since DataParallel
    124.  
    125.     checkpoint = torch.load(this_weights)
    126.     checkpoint = checkpoint['state_dict']
    127.  
    128.     # base_dict = {('base_model.' + k).replace('base_model.fc', 'new_fc'): v for k, v in list(checkpoint.items())}
    129.     base_dict = {'.'.join(k.split('.')[1:]): v for k, v in list(checkpoint.items())}
    130.     replace_dict = {'base_model.classifier.weight': 'new_fc.weight',
    131.                     'base_model.classifier.bias': 'new_fc.bias',
    132.                     }
    133.     for k, v in replace_dict.items():
    134.         if k in base_dict:
    135.             base_dict[v] = base_dict.pop(k)
    136.  
    137.     net.load_state_dict(base_dict)
    138.  
    139.     input_size = net.scale_size if args.full_res else net.input_size
    140.     if args.test_crops == 1:
    141.         cropping = torchvision.transforms.Compose([
    142.             GroupScale(net.scale_size),
    143.             GroupCenterCrop(input_size),
    144.         ])
    145.     elif args.test_crops == 3:  # do not flip, so only 5 crops
    146.         cropping = torchvision.transforms.Compose([
    147.             GroupFullResSample(input_size, net.scale_size, flip=False)
    148.         ])
    149.     elif args.test_crops == 5:  # do not flip, so only 5 crops
    150.         cropping = torchvision.transforms.Compose([
    151.             GroupOverSample(input_size, net.scale_size, flip=False)
    152.         ])
    153.     elif args.test_crops == 10:
    154.         cropping = torchvision.transforms.Compose([
    155.             GroupOverSample(input_size, net.scale_size)
    156.         ])
    157.     else:
    158.         raise ValueError("Only 1, 5, 10 crops are supported while we got {}".format(args.test_crops))
    159.  
    160.     data_loader = torch.utils.data.DataLoader(
    161.             TSNDataSet(root_path, test_file if test_file is not None else val_list, num_segments=this_test_segments,
    162.                        new_length=1 if modality == "RGB" else 5,
    163.                        modality=modality,
    164.                        image_tmpl=prefix,
    165.                        test_mode=True,
    166.                        remove_missing=len(weights_list) == 1,
    167.                        transform=torchvision.transforms.Compose([
    168.                            cropping,
    169.                            Stack(roll=(this_arch in ['BNInception', 'InceptionV3'])),
    170.                            ToTorchFormatTensor(div=(this_arch not in ['BNInception', 'InceptionV3'])),
    171.                            GroupNormalize(net.input_mean, net.input_std),
    172.                        ]), dense_sample=args.dense_sample, twice_sample=args.twice_sample),
    173.             batch_size=args.batch_size, shuffle=False,
    174.             num_workers=args.workers, pin_memory=True,
    175.     )
    176.  
    177.     if args.gpus is not None:
    178.         devices = [args.gpus[i] for i in range(args.workers)]
    179.     else:
    180.         devices = list(range(args.workers))
    181.  
    182.     net = torch.nn.DataParallel(net.cuda())
    183.     net.eval()
    184.  
    185.     data_gen = enumerate(data_loader)
    186.  
    187.     if total_num is None:
    188.         total_num = len(data_loader.dataset)
    189.     else:
    190.         assert total_num == len(data_loader.dataset)
    191.  
    192.     data_iter_list.append(data_gen)
    193.     net_list.append(net)
    194.  
    195.  
    196. output = []
    197.  
    198.  
    199. def eval_video(video_data, net, this_test_segments, modality):
    200.     net.eval()
    201.     with torch.no_grad():
    202.         i, data, label = video_data
    203.         batch_size = label.numel()
    204.         num_crop = args.test_crops
    205.         if args.dense_sample:
    206.             num_crop *= 10  # 10 clips for testing when using dense sample
    207.  
    208.         if args.twice_sample:
    209.             num_crop *= 2
    210.  
    211.         if modality == 'RGB':
    212.             length = 3
    213.         elif modality == 'Flow':
    214.             length = 10
    215.         elif modality == 'RGBDiff':
    216.             length = 18
    217.         else:
    218.             raise ValueError("Unknown modality "+ modality)
    219.  
    220.         data_in = data.view(-1, length, data.size(2), data.size(3))
    221.         if is_shift:
    222.             data_in = data_in.view(batch_size * num_crop, this_test_segments, length, data_in.size(2), data_in.size(3))
    223.         rst = net(data_in)
    224.         rst = rst.reshape(batch_size, num_crop, -1).mean(1)
    225.  
    226.         if args.softmax:
    227.             # take the softmax to normalize the output to probability
    228.             rst = F.softmax(rst, dim=1)
    229.  
    230.         rst = rst.data.cpu().numpy().copy()
    231.  
    232.         if net.module.is_shift:
    233.             rst = rst.reshape(batch_size, num_class)
    234.         else:
    235.             rst = rst.reshape((batch_size, -1, num_class)).mean(axis=1).reshape((batch_size, num_class))
    236.  
    237.         return i, rst, label
    238.  
    239.  
    240. proc_start_time = time.time()
    241. max_num = args.max_num if args.max_num > 0 else total_num
    242.  
    243. top1 = AverageMeter()
    244. top5 = AverageMeter()
    245.  
    246. for i, data_label_pairs in enumerate(zip(*data_iter_list)):
    247.     with torch.no_grad():
    248.         if i >= max_num:
    249.             break
    250.         this_rst_list = []
    251.         this_label = None
    252.         for n_seg, (_, (data, label)), net, modality in zip(test_segments_list, data_label_pairs, net_list, modality_list):
    253.             rst = eval_video((i, data, label), net, n_seg, modality)
    254.             this_rst_list.append(rst[1])
    255.             this_label = label
    256.         assert len(this_rst_list) == len(coeff_list)
    257.         for i_coeff in range(len(this_rst_list)):
    258.             this_rst_list[i_coeff] *= coeff_list[i_coeff]
    259.         ensembled_predict = sum(this_rst_list) / len(this_rst_list)
    260.  
    261.         for p, g in zip(ensembled_predict, this_label.cpu().numpy()):
    262.             output.append([p[None, ...], g])
    263.         cnt_time = time.time() - proc_start_time
    264.         prec1, prec5 = accuracy(torch.from_numpy(ensembled_predict), this_label, topk=(1, 5))
    265.         top1.update(prec1.item(), this_label.numel())
    266.         top5.update(prec5.item(), this_label.numel())
    267.         if i % 20 == 0:
    268.             print('video {} done, total {}/{}, average {:.3f} sec/video, '
    269.                   'moving Prec@1 {:.3f} Prec@5 {:.3f}'.format(i * args.batch_size, i * args.batch_size, total_num,
    270.                                                               float(cnt_time) / (i+1) / args.batch_size, top1.avg, top5.avg))
    271.  
    272. video_pred = [np.argmax(x[0]) for x in output]
    273. video_pred_top5 = [np.argsort(np.mean(x[0], axis=0).reshape(-1))[::-1][:5] for x in output]
    274.  
    275. video_labels = [x[1] for x in output]
    276.  
    277.  
    278. if args.csv_file is not None:
    279.     print('=> Writing result to csv file: {}'.format(args.csv_file))
    280.     with open(test_file_list[0].replace('test_videofolder.txt', 'category.txt')) as f:
    281.         categories = f.readlines()
    282.     categories = [f.strip() for f in categories]
    283.     with open(test_file_list[0]) as f:
    284.         vid_names = f.readlines()
    285.     vid_names = [n.split(' ')[0] for n in vid_names]
    286.     assert len(vid_names) == len(video_pred)
    287.     if args.dataset != 'somethingv2':  # only output top1
    288.         with open(args.csv_file, 'w') as f:
    289.             for n, pred in zip(vid_names, video_pred):
    290.                 f.write('{};{}\n'.format(n, categories[pred]))
    291.     else:
    292.         with open(args.csv_file, 'w') as f:
    293.             for n, pred5 in zip(vid_names, video_pred_top5):
    294.                 fill = [n]
    295.                 for p in list(pred5):
    296.                     fill.append(p)
    297.                 f.write('{};{};{};{};{};{}\n'.format(*fill))
    298.  
    299.  
    300. cf = confusion_matrix(video_labels, video_pred).astype(float)
    301.  
    302. np.save('cm.npy', cf)
    303. cls_cnt = cf.sum(axis=1)
    304. cls_hit = np.diag(cf)
    305.  
    306. cls_acc = cls_hit / cls_cnt
    307. print(cls_acc)
    308. upper = np.mean(np.max(cf, axis=1) / cls_cnt)
    309. print('upper bound: {}'.format(upper))
    310.  
    311. print('-----Evaluation is finished------')
    312. print('Class Accuracy {:.02f}%'.format(np.mean(cls_acc) * 100))
    313. print('Overall Prec@1 {:.02f}% Prec@5 {:.02f}%'.format(top1.avg, top5.avg))
    314.  
    315.

    创作不易 觉得有帮助请点赞关注收藏~~~

  • 相关阅读:
    算法竞赛进阶指南——链表学习笔记
    HTML+CSS+JS鲜花商城网页设计期末课程大作业 web前端开发技术 web课程设计 网页规划与设计
    北大学子荣获SRC全球总决赛本科生第一名！完美世界被曝开 17800 元“付费上班”项目；苹果和安卓有望统一充电接口|极客头条
    卸载各种方式安装的K8S集群
    最新WooCommerce教程指南-如何搭建B2C外贸独立站
    企业门户的必备选择，WorkPlus的定制化解决方案
    基于Jsp和Servlet的简单项目
    vue3+ts项目04-国际化
    建筑模板的成本如何控制?
    B2B企业如何打造独立站：从策略到实施的全面指南
  • 原文地址：https://blog.csdn.net/jiebaoshayebuhui/article/details/127856224