• ECCV2022细粒度图像检索SEMICON代码学习记录

    代码链接：GitHub - aassxun/SEMICON

    环境配置

    1. # 创建&激活虚拟环境
    2. conda create -n semicon python==3.8.5
    3. conda activate semicon
    4.  
    5. # 安装相关依赖包 (该 pytorch 为无 gpu 版本)
    6. conda install pytorch==1.10.0 torchvision==0.11.1 torchaudio==0.10.0 cpuonly -c pytorch
    7. pip install numpy==1.19.2
    8. pip install loguru==0.5.3
    9. pip install tqdm==4.54.1
    10. pip install pandas
    11. pip install scipy

    需要将 SEMICON_train.py、SEMICON.py、Hash_mAP.py、baseline_train.py、baseline.py 中的import models.resnet as resnet 和 from models.resnet import *改为 import models.resnet_torch as resnet 和 from models.resnet_torch import *

    下载CUB_200_2011数据集

    参考博客：CUB-200-2011鸟类数据集的下载与使用pytorch加载_景唯acr的博客-CSDN博客_cub200-2011

    代码运行

    1）训练

    python run.py --dataset cub-2011 --root /dataset/CUB2011/CUB_200_2011 --max-epoch 30 --gpu 0 --arch semicon --batch-size 16 --max-iter 40 --code-length 12,24,32,48 --lr 2.5e-4 --wd 1e-4 --optim SGD --lr-step 40 --num-samples 2000 --info 'CUB-SEMICON' --momen=0.91

    2）测试

    python run.py --dataset cub-2011 --root /dataset/CUB2011/CUB_200_2011 --gpu 0 --arch test --batch-size 16 --code-length 12,24,32,48 --wd 1e-4 --info 'CUB-SEMICON'

    如果不想使用 gpu，将参数 --gpu 设为False 即可

    代码学习

    1）固定随机种子

    与 YOLO-X 类似，将随机种子进行固定，后续实验将在此固定的随机种子下进行 (如消融实验等)，增强了模型的可复现性 (但我觉得也只是仅限于特定的随机数，换另一个随机数可能结果又不一样了)。

    torch.backends.cudnn.deterministic 和 torch.backends.cudnn.benchmark：前者可以保证每次运行网络的时候相同输入的输出是固定的，后者为整个网络的每个卷积层搜索最适合它的卷积实现算法，进而实现网络的加速。适用场景是网络结构固定，网络的输入形状（包括 batch size，图片大小，输入的通道）是不变的，其实也就是一般情况下都比较适用。反之，如果卷积层的设置一直变化，将会导致程序不停地做优化，反而会耗费更多的时间。

    参考博客：

    【pytorch】torch.backends.cudnn.deterministic_Xhfei1224的博客-CSDN博客_torch.backends.cudnn.deter

    torch.backends.cudnn.benchmark_Wanderer001的博客-CSDN博客_torch.backends.cudnn.benchmark

    1. def seed_everything(seed):
    2.     random.seed(seed)
    3.     os.environ['PYTHONHASHSEED'] = str(seed)
    4.     np.random.seed(seed)
    5.     torch.manual_seed(seed)
    6.     torch.cuda.manual_seed(seed)
    7.     torch.backends.cudnn.deterministic = True
    8.     torch.backends.cudnn.benchmark = True
    9.  
    10. seed_everything(68)

    2）数据加载

    对应脚本：data/cub_2011.py

    函数 load_data() 会返回 3 个 dataloader：query_dataloader、train_dataloader、retrieval_dataloader

    1. # 划分训练、测试集
    2. Cub2011.init(root)
    3. # 定义查询、训练及检索数据集，涉及的数据增强在 data/transform.py 中
    4. query_dataset = Cub2011(root, 'query', query_transform())
    5. train_dataset = Cub2011(root, 'train', train_transform())
    6. retrieval_dataset = Cub2011(root, 'retrieval', query_transform())
    1. class Cub2011(Dataset):
    2.  
    3.     def __init__(self, root, mode, transform=None, loader=default_loader):
    4.         self.root = os.path.expanduser(root)
    5.         self.transform = transform
    6.         self.loader = default_loader
    7.  
    8.         if mode == 'train':
    9.             self.data = Cub2011.TRAIN_DATA
    10.             self.targets = Cub2011.TRAIN_TARGETS
    11.         elif mode == 'query':
    12.             self.data = Cub2011.QUERY_DATA
    13.             self.targets = Cub2011.QUERY_TARGETS
    14.         elif mode == 'retrieval':
    15.             self.data = Cub2011.RETRIEVAL_DATA
    16.             self.targets = Cub2011.RETRIEVAL_TARGETS
    17.         else:
    18.             raise ValueError(r'Invalid arguments: mode, can\'t load dataset!')
    19.  
    20.     @staticmethod
    21.     def init(root):
    22.         images = pd.read_csv(os.path.join(root, 'images.txt'), sep=' ',
    23.                              names=['img_id', 'filepath'])
    24.         image_class_labels = pd.read_csv(os.path.join(root, 'image_class_labels.txt'),
    25.                                          sep=' ', names=['img_id', 'target'])
    26.         train_test_split = pd.read_csv(os.path.join(root, 'train_test_split.txt'),
    27.                                        sep=' ', names=['img_id', 'is_training_img'])
    28.  
    29.         data = images.merge(image_class_labels, on='img_id')
    30.         all_data = data.merge(train_test_split, on='img_id')
    31.         all_data['filepath'] = 'images/' + all_data['filepath']
    32.         train_data = all_data[all_data['is_training_img'] == 1]
    33.         test_data = all_data[all_data['is_training_img'] == 0]
    34.  
    35.         # Split dataset
    36.         Cub2011.QUERY_DATA = test_data['filepath'].to_numpy()
    37.         Cub2011.QUERY_TARGETS = encode_onehot((test_data['target'] - 1).tolist(), 200)
    38.  
    39.         Cub2011.TRAIN_DATA = train_data['filepath'].to_numpy()
    40.         Cub2011.TRAIN_TARGETS = encode_onehot((train_data['target'] - 1).tolist(), 200)
    41.  
    42.         Cub2011.RETRIEVAL_DATA = train_data['filepath'].to_numpy()
    43.         Cub2011.RETRIEVAL_TARGETS = encode_onehot((train_data['target'] - 1).tolist(), 200)
    44.  
    45.     def get_onehot_targets(self):
    46.         return torch.from_numpy(self.targets).float()
    47.  
    48.     def __len__(self):
    49.         return len(self.data)
    50.  
    51.     def __getitem__(self, idx):
    52.         img = Image.open(os.path.join(self.root, self.data[idx])).convert('RGB')
    53.  
    54.         if self.transform is not None:
    55.             img = self.transform(img)
    56.  
    57.         return img, self.targets[idx], idx

    3）网络训练

    主干网络

    这里只使用了 resnet50 的前三个 layer，具体可查看 models/SEMICON.py 中的 ResNet_Backbone 类

    model = ResNet_Backbone(Bottleneck, [3, 4, 6], **kwargs)

    全局/局部转换网络

    1. class ResNet_Refine(nn.Module):
    2.  
    3.     def __init__(self, block, layer, is_local=True, num_classes=1000, zero_init_residual=False,
    4.                  groups=1, width_per_group=64, norm_layer=None):
    5.         super(ResNet_Refine, self).__init__()
    6.         if norm_layer is None:
    7.             norm_layer = nn.BatchNorm2d
    8.         self._norm_layer = norm_layer
    9.  
    10.         self.inplanes = 1024
    11.         self.dilation = 1
    12.  
    13.         self.is_local = is_local
    14.         self.groups = groups
    15.         self.base_width = width_per_group
    16.         self.layer4 = self._make_layer(block, 512, layer, stride=2)
    17.         self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
    18.  
    19.         for m in self.modules():
    20.             if isinstance(m, nn.Conv2d):
    21.                 nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
    22.             elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
    23.                 nn.init.constant_(m.weight, 1)
    24.                 nn.init.constant_(m.bias, 0)
    25.  
    26.         # Zero-initialize the last BN in each residual branch,
    27.         # so that the residual branch starts with zeros, and each residual block behaves like an identity.
    28.         # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
    29.         if zero_init_residual:
    30.             for m in self.modules():
    31.                 if isinstance(m, Bottleneck):
    32.                     nn.init.constant_(m.bn3.weight, 0)
    33.                 elif isinstance(m, BasicBlock):
    34.                     nn.init.constant_(m.bn2.weight, 0)
    35.  
    36.     def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
    37.         norm_layer = self._norm_layer
    38.         downsample = None
    39.         previous_dilation = self.dilation
    40.         if dilate:
    41.             self.dilation *= stride
    42.             stride = 1
    43.         if stride != 1 or self.inplanes != planes * block.expansion:
    44.             downsample = nn.Sequential(
    45.                 conv1x1(self.inplanes, planes * block.expansion, stride),
    46.                 norm_layer(planes * block.expansion),
    47.             )
    48.  
    49.         layers = []
    50.         layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
    51.                             self.base_width, previous_dilation, norm_layer))
    52.         self.inplanes = planes * block.expansion
    53.         for _ in range(1, blocks):
    54.             layers.append(block(self.inplanes, planes, groups=self.groups,
    55.                                 base_width=self.base_width, dilation=self.dilation,
    56.                                 norm_layer=norm_layer))
    57.             layers.append(ChannelTransformer(planes * block.expansion, max(planes * block.expansion // 64, 16)))
    58.         return nn.Sequential(*layers)
    59.  
    60.     def _forward_impl(self, x):
    61.         x = self.layer4(x)
    62.  
    63.         pool_x = self.avgpool(x)
    64.         pool_x = torch.flatten(pool_x, 1)
    65.         if self.is_local:
    66.             return x, pool_x
    67.         else:
    68.             return pool_x
    69.  
    70.     def forward(self, x):
    71.         return self._forward_impl(x)

    SEM

    1. class SEM(nn.Module):
    2.  
    3.     def __init__(self, block, layer, att_size=4, num_classes=1000, zero_init_residual=False,
    4.                  groups=1, width_per_group=64, replace_stride_with_dilation=None,
    5.                  norm_layer=None):
    6.         super(SEM, self).__init__()
    7.         if norm_layer is None:
    8.             norm_layer = nn.BatchNorm2d
    9.         self._norm_layer = norm_layer
    10.  
    11.         self.inplanes = 1024
    12.         self.dilation = 1
    13.         self.att_size = att_size
    14.         if replace_stride_with_dilation is None:
    15.             # each element in the tuple indicates if we should replace
    16.             # the 2x2 stride with a dilated convolution instead
    17.             replace_stride_with_dilation = [False, False, False]
    18.         if len(replace_stride_with_dilation) != 3:
    19.             raise ValueError("replace_stride_with_dilation should be None "
    20.                              "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
    21.         self.groups = groups
    22.         self.base_width = width_per_group
    23.  
    24.         self.layer4 = self._make_layer(block, 512, layer, stride=1)
    25.  
    26.         self.feature1 = nn.Sequential(
    27.             conv1x1(self.inplanes, 1),
    28.             nn.BatchNorm2d(1),
    29.             nn.ReLU(inplace=True),
    30.         )
    31.         self.feature2 = nn.Sequential(
    32.             conv1x1(self.inplanes, 1),
    33.             nn.BatchNorm2d(1),
    34.             nn.ReLU(inplace=True)
    35.         )
    36.         self.feature3 = nn.Sequential(
    37.             conv1x1(self.inplanes, 1),
    38.             nn.BatchNorm2d(1),
    39.             nn.ReLU(inplace=True)
    40.         )
    41.         
    42.         for m in self.modules():
    43.             if isinstance(m, nn.Conv2d):
    44.                 nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
    45.             elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
    46.                 nn.init.constant_(m.weight, 1)
    47.                 nn.init.constant_(m.bias, 0)
    48.  
    49.         # Zero-initialize the last BN in each residual branch,
    50.         # so that the residual branch starts with zeros, and each residual block behaves like an identity.
    51.         # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
    52.         if zero_init_residual:
    53.             for m in self.modules():
    54.                 if isinstance(m, Bottleneck):
    55.                     nn.init.constant_(m.bn3.weight, 0)
    56.                 elif isinstance(m, BasicBlock):
    57.                     nn.init.constant_(m.bn2.weight, 0)
    58.  
    59.     def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
    60.         norm_layer = self._norm_layer
    61.         downsample = None
    62.         previous_dilation = self.dilation
    63.         att_expansion = 0.25
    64.         layers = []
    65.         layers.append(block(self.inplanes, int(self.inplanes * att_expansion), stride,
    66.                             downsample, self.groups, self.base_width, previous_dilation, norm_layer))
    67.         for _ in range(1, blocks):
    68.             layers.append(nn.Sequential(
    69.                 conv1x1(self.inplanes, int(self.inplanes * att_expansion)),
    70.                 nn.BatchNorm2d(int(self.inplanes * att_expansion))
    71.             ))
    72.             self.inplanes = int(self.inplanes * att_expansion)
    73.             layers.append(block(self.inplanes, int(self.inplanes * att_expansion), groups=self.groups,
    74.                                 base_width=self.base_width, dilation=self.dilation, norm_layer=norm_layer))
    75.         return nn.Sequential(*layers)
    76.  
    77.     def _mask(self, feature, x):
    78.         with torch.no_grad():
    79.             cam1 = feature.mean(1)
    80.             attn = torch.softmax(cam1.view(x.shape[0], x.shape[2] * x.shape[3]), dim=1)#B,H,W
    81.             std, mean = torch.std_mean(attn)
    82.             attn = (attn - mean) / (std ** 0.3) + 1 #0.15
    83.             attn = (attn.view((x.shape[0], 1, x.shape[2], x.shape[3]))).clamp(0, 2)
    84.         return attn
    85.  
    86.     def _forward_impl(self, x):
    87.         x = self.layer4(x)#bs*64*14*14
    88.         fea1 = self.feature1(x) #bs*1*14*14
    89.         attn = 2-self._mask(fea1, x)
    90.  
    91.         x = x.mul(attn.repeat(1, self.inplanes, 1, 1))
    92.         fea2 = self.feature2(x)
    93.         attn = 2-self._mask(fea2, x)
    94.  
    95.         x = x.mul(attn.repeat(1, self.inplanes, 1, 1))
    96.         fea3 = self.feature3(x)
    97.  
    98.         x = torch.cat([fea1,fea2,fea3], dim=1)
    99.         return x
    100.  
    101.     def forward(self, x):
    102.         return self._forward_impl(x)

    ICON

    1. class ChannelTransformer(nn.Module):
    2.     def __init__(self, dim, num_heads):
    3.         super().__init__()
    4.         self.num_heads = num_heads
    5.         head_dim = dim // num_heads
    6.         self.scale = head_dim ** -0.5
    7.         self.head_dim = head_dim
    8.         self.norm = nn.BatchNorm2d(dim)
    9.         self.relu = nn.ReLU(inplace=True)
    10.  
    11.         self.qkv = nn.Conv2d(dim, dim * 3, 1, groups=num_heads)
    12.         self.qkv2 = nn.Conv2d(dim, dim * 3, 1, groups=head_dim)
    13.  
    14.     def forward(self, x):
    15.         B, C, H, W = x.shape
    16.         qkv = self.qkv(x).reshape(B, 3, self.num_heads, self.head_dim, H * W).transpose(0, 1)
    17.         q, k, v = qkv[0], qkv[1], qkv[2]
    18.  
    19.         attn = (q @ k.transpose(-2, -1)) * self.scale
    20.         attn = torch.sign(attn) * torch.sqrt(torch.abs(attn) + 1e-5)
    21.         attn = attn.softmax(dim=-1)
    22.  
    23.         x = ((attn @ v).reshape(B, C, H, W) + x).reshape(B, self.num_heads, self.head_dim, H, W).transpose(1, 2).reshape(B, C, H, W)
    24.         y = self.norm(x)
    25.         x = self.relu(y)
    26.         qkv2 = self.qkv2(x).reshape(B, 3, self.head_dim, self.num_heads, H * W).transpose(0, 1)
    27.         q, k, v = qkv2[0], qkv2[1], qkv2[2]
    28.  
    29.         attn = (q @ k.transpose(-2, -1)) * (self.num_heads ** -0.5)
    30.         attn = torch.sign(attn) * torch.sqrt(torch.abs(attn) + 1e-5)
    31.         attn = attn.softmax(dim=-1)
    32.         x = (attn @ v).reshape(B, self.head_dim, self.num_heads, H, W).transpose(1, 2).reshape(B, C, H, W) + y
    33.         return x

    损失函数

    1. class ADSH_Loss(nn.Module):
    2.     def __init__(self, code_length, gamma):
    3.         super(ADSH_Loss, self).__init__()
    4.         self.code_length = code_length
    5.         self.gamma = gamma
    6.  
    7.     def forward(self, F, B, S, omega):
    8.         hash_loss = ((self.code_length * S - F @ B.t()) ** 2).sum() / (F.shape[0] * B.shape[0]) / self.code_length * 12
    9.         quantization_loss = ((F - B[omega, :]) ** 2).sum() / (F.shape[0] * B.shape[0]) * self.gamma / self.code_length * 12
    10.  
    11.         loss = hash_loss + quantization_loss
    12.         return loss, hash_loss, quantization_loss

    4）网络测试

    1. def valid(query_dataloader, train_dataloader, retrieval_dataloader, code_length, args):
    2.     num_classes, att_size, feat_size = args.num_classes, 1, 2048
    3.     model = SEMICON.semicon(code_length=code_length, num_classes=num_classes, att_size=att_size, feat_size=feat_size,
    4.                             device=args.device, pretrained=True)
    5.  
    6.     model.to(args.device)
    7.     
    8.     model.load_state_dict(torch.load('./checkpoints/' + args.info + '/' + str(code_length) + '/model.pkl'), strict=False)
    9.     model.eval()
    10.     query_code = torch.load('./checkpoints/' + args.info + '/' + str(code_length) + '/query_code.t')
    11.     query_code = query_code.to(args.device)
    12.     query_dataloader.dataset.get_onehot_targets = torch.load('./checkpoints/' + args.info + '/' + str(code_length) + '/query_targets.t')
    13.     B = torch.load('./checkpoints/' + args.info + '/' + str(code_length) + '/database_code.t')
    14.     B = B.to(args.device)
    15.     retrieval_targets = torch.load('./checkpoints/' + args.info + '/' + str(code_length) + '/database_targets.t')
    16.     retrieval_targets = retrieval_targets.to(args.device)
    17.  
    18.     
    19.     mAP = evaluate.mean_average_precision(
    20.         query_code.to(args.device),
    21.         B,
    22.         query_dataloader.dataset.get_onehot_targets().to(args.device),
    23.         retrieval_targets,
    24.         args.device,
    25.         args.topk,
    26.     )
    27.     print("Code_Length: " + str(code_length), end="; ")
    28.     print('[mAP:{:.5f}]'.format(mAP))

    5）网络结构 (onnx model)

  • 相关阅读:
    web页面之间的3种关系
    分享：互信息在对比学习中的应用
    实体-联系模型--E-R图
    Django+Celery框架自动化定时任务开发
    VisualStudio(VS)设置程序的版本信息(C-C++)
    学习笔记-Power-Linux
    微星迫击炮b660m使用intel arc a750/770显卡功耗优化方法
    Python语言学习：Python语言学习之逻辑控制语句(if语句&for语句&while语句&range语句&with语句)的简介、案例应用之详细攻略
    JVM学习第一天
    【产线故障】线上接口请求过慢如何排查？
  • 原文地址：https://blog.csdn.net/qq_38964360/article/details/126922099