• 使用CIFAR10数据集完成知识蒸馏（参照了快乐就好_Peng博主的博客）致谢

           俺这次做的是知识蒸馏，但是看到别人做的数据集不是CIFAT10，所以有点手痒，就自己做了一期CIFAR10数据集的尝试，做出来的结果........还行，还能感觉到教师网络确实是具有指导作用的

    在这里我推荐搭建去B站看一下：同济子豪兄讲解的知识蒸馏，挺详细的

    话不多说，直接进入正题吧：

    1.先进行一个教师网络的定义，并且这个网络的隐藏层的参数我设置为1200，这个主要是为了凸显教师网络和学生网络的参数量大小之间具有巨大的差距。

    注意：在运行我的代码的时候可能会报错关于：if __name__ == "__main__": 问题，这个百度就行了，这里就不详细介绍了，因为我为了调用教师网络去到蒸馏网络那里运行，我注释了if __name__ == "__main__"语句，并且把缩径也关掉了，但是你直接使用教师网络运行的话会报错的可能。。。。。。。。。。

    1. # -*- coding:utf-8 -*-
    2. # @Time : 2022-06-27 13:43
    3. # @Author : DaFuChen
    4. # @File : teacher_moudle.py
    5. # @software: PyCharm
    6.  
    7.  
    8. # if __name__ == "__main__":
    9.  
    10. import torch
    11. import torch.nn as nn
    12. import torchvision
    13. import torchvision.transforms as transforms
    14. from tqdm import tqdm  # tqdm 是python中很常用的模块,它的作用就是在终端上出现一个进度条,使得代码进度可视化.
    15.  
    16. file_name_path = "test_teacher" + ".txt"
    17. file_name_path = open(file_name_path, "w")
    18.  
    19.  
    20.  
    21. """
    22. 第一部分
    23. 开始进行数据集合的引入
    24. """
    25.  
    26. # 设置一个随机种子，为了实验的复现
    27. torch.manual_seed(9)
    28. device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    29.  
    30. # 使用cudnn加速卷积运算
    31. torch.backends.cudnn.benchmark = True
    32.  
    33. from torchvision.transforms.transforms import ToTensor
    34.  
    35. # 载入训练集
    36. trainset = torchvision.datasets.CIFAR10(
    37.     root='./datac10',
    38.     train=True,
    39.     transform=ToTensor(),  # 将PLIImage转化为适合pytorch进行处理的数据格式
    40.     download=True
    41. )
    42.  
    43. # 生成测试集合数据
    44. testset = torchvision.datasets.CIFAR10(
    45.     root='./datac10',
    46.     train=False,
    47.     transform=transforms.ToTensor(),
    48.     download=True
    49. )
    50.  
    51. tranloader = torch.utils.data.DataLoader(
    52.     dataset=trainset,
    53.     batch_size=32,
    54.     shuffle=True
    55. )
    56.  
    57. testloader = torch.utils.data.DataLoader(
    58.     dataset=testset,
    59.     batch_size=32,
    60.     shuffle=False
    61. )
    62.  
    63. """
    64. 第二部分
    65. 定义teacher模型
    66. """
    67.  
    68.  
    69. class teacherMoudle(nn.Module):
    70.     def __init__(self, in_channels=1, num_class=10):
    71.         super(teacherMoudle, self).__init__()
    72.  
    73.         # 定义卷积
    74.         self.conv1 = nn.Conv2d(3, 6, 5)
    75.         self.conv2 = nn.Conv2d(6, 16, 5)
    76.  
    77.         # 定义池化层
    78.         self.pool = nn.MaxPool2d(2, 2)
    79.  
    80.         # 定义激活函数选择relu()
    81.         self.relu = nn.ReLU()
    82.  
    83.         # 定义droput层
    84.         self.droput = nn.Dropout(p=0.5)
    85.  
    86.         # 定义全连接层
    87.         self.fc1 = nn.Linear(16 * 5 * 5, 1200)
    88.         # 这里需要注意的是教师的模型的隐藏层的参数很对,有1200个
    89.         self.fc2 = nn.Linear(1200, 600)
    90.         self.fc3 = nn.Linear(600, num_class)
    91.  
    92.     def forward(self, x):
    93.         x = self.pool(self.relu(self.conv1(x)))
    94.         x = self.pool(self.relu(self.conv2(x)))
    95.  
    96.         x = x.view(-1, 16 * 5 * 5)
    97.         x = self.fc1(x)
    98.         x = self.droput(x)
    99.         x = self.relu(x)
    100.  
    101.         x = self.fc2(x)
    102.         x = self.droput(x)
    103.         x = self.relu(x)
    104.  
    105.         x = self.fc3(x)
    106.  
    107.         return x
    108.  
    109.  
    110. """
    111. 第三部分
    112. 训练教师模型
    113. """
    114. model = teacherMoudle()
    115. model = model.to(device)
    116.  
    117. # 定义损失函数和优化器
    118. crossLoss = nn.CrossEntropyLoss()
    119. optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)
    120.  
    121. epochs = 6
    122. for epoch in range(epochs):
    123.     model.train()
    124.     # model.train()的作用是启用
    125.     # Batch Normalization和Dropout。如果模型中有BN层或Dropout层, model.train()
    126.     # 是保证训练时BN层能够用到每一批数据的均值和方差, 对于Dropout, model.train()
    127.     # 是随机取一部分网络连接来训练更新参数。”
    128.  
    129.  
    130.     for image, lables in tqdm(tranloader):
    131.         image = image.to(device)
    132.         lables = lables.to(device)
    133.         running_loss = 0.0
    134.  
    135.         # 前向预测
    136.         pred = model(image)
    137.         loss = crossLoss(pred, lables)
    138.         running_loss += loss.item()
    139.  
    140.         # 步骤
    141.         # 1.优化器清零
    142.         # 2.损失返回
    143.         # 3.优化器更新参数值
    144.  
    145.         # 反向传播
    146.         optimizer.zero_grad()  # 必须进行的一个清零操作
    147.  
    148.         # ##################################重要的地方 需要利用loss进行反向传播进行更新参数，不然就会没有优化这个模型一样
    149.         loss.backward()
    150.  
    151.         optimizer.step()  # 整体参数的保存
    152.  
    153.     """
    154.     第四部分
    155.     模型的评估
    156.     """
    157.     # 在评估模式下，batchNorm层，dropout层等用于优化训练而添加的网络层会被关闭，从而使得评估时不会发生偏移。
    158.     model.eval()
    159.     num_correct = 0
    160.     num_samples = 0
    161.  
    162.     with torch.no_grad():
    163.         for image, lables in testloader:
    164.             image = image.to(device)
    165.             lables = lables.to(device)
    166.  
    167.             pred = model(image)
    168.  
    169.             predictions = pred.max(1).indices  # 获取标签值
    170.             num_correct += (predictions == lables).sum()
    171.             num_samples += predictions.size(0)
    172.         acc = (num_correct / num_samples).item()
    173.  
    174.         model.train()
    175.  
    176.  
    177.         print('Epoch:{}\t Accuracy:{:.5f}'.format(epoch + 1, acc))
    178.         print('loss : ' + str(running_loss))
    179.  
    180.         txt = open("./test_teacher.txt", "a").write((str(epoch+1) + "    " + str(acc) + "    " + str(running_loss) + '\r\n'))
    181.  
    182.

    训练的结果是：

    1    0.49399998784065247    1.1463124752044678

2    0.5374000072479248    1.0659717321395874

3    0.5343999862670898    1.388565182685852

4    0.5910999774932861    1.176950454711914

5    0.6060000061988831    1.2053321599960327

6    0.6089999675750732    1.2049661874771118

    接下来就是学生网络了，我在隐藏层那里定义了相对于教师网络来说隐藏层还是很少的神经元，所以看到了学习的效果果真不如教师网络......

    1. # -*- coding:utf-8 -*-
    2. # @Time : 2022-06-27 13:44
    3. # @Author : DaFuChen
    4. # @File : student_moudle.py
    5. # @software: PyCharm
    6. import torchvision.datasets
    7.  
    8. if __name__ == "__main__":
    9.  
    10.     import torch
    11.     import torch.nn as nn
    12.     from tqdm import tqdm
    13.     from torchvision.transforms.transforms import ToTensor
    14.  
    15.  
    16.  
    17.     file_name_path = "test_student" + ".txt"
    18.     file_name_path = (file_name_path, "w")
    19.  
    20.     torch.manual_seed(8)
    21.     device = torch.device("cuda" if torch.cuda.is_available()  else "cpu")
    22.     torch.backends.cudnn.benchmark = True
    23.  
    24.     train_set = torchvision.datasets.CIFAR10(
    25.         root="./datac10",
    26.         train=True,
    27.         transform=ToTensor(),
    28.         download=True
    29.     )
    30.  
    31.     test_set = torchvision.datasets.CIFAR10(
    32.         root="./datac10",
    33.         train=False,
    34.         transform=ToTensor(),
    35.         download=True
    36.     )
    37.  
    38.     train_loader = torch.utils.data.DataLoader(
    39.         dataset=train_set,
    40.         batch_size=32,
    41.         shuffle=True
    42.     )
    43.  
    44.     test_loader = torch.utils.data.DataLoader(
    45.         dataset=test_set,
    46.         batch_size=32,
    47.         shuffle=True
    48.     )
    49.  
    50.  
    51.     class student_moudle(nn.Module):
    52.         def __init__(self, in_channels=1, num_class=10):
    53.             super(student_moudle, self).__init__()
    54.  
    55.             self.conv1 = nn.Conv2d(3, 6, 5)
    56.             self.conv2 = nn.Conv2d(6, 16, 5)
    57.  
    58.             self.pool = nn.MaxPool2d(2, 2)
    59.             self.relu = nn.ReLU()
    60.             self.droput = nn.Dropout(p=0.5)
    61.  
    62.             # 定义全连接层
    63.             self.fc1 = nn.Linear(16 * 5 * 5, 64)
    64.             # 学生的参数是64层，比教师网络的层数少很多，可以使用
    65.             self.fc2 = nn.Linear(64, 32)
    66.             self.fc3 = nn.Linear(32, num_class)
    67.  
    68.  
    69.         def forward(self, x):
    70.             x = self.pool(self.relu(self.conv1(x)))
    71.             x = self.pool(self.relu(self.conv2(x)))
    72.  
    73.             x = x.view(-1, 16 * 5 * 5)
    74.             x = self.fc1(x)
    75.             x = self.droput(x)
    76.             x = self.relu(x)
    77.  
    78.             x = self.fc2(x)
    79.             x = self.droput(x)
    80.             x = self.relu(x)
    81.  
    82.             x = self.fc3(x)
    83.  
    84.             return x
    85.  
    86.     model = student_moudle()
    87.     model = model.to(device)
    88.  
    89.     crossLoss = nn.CrossEntropyLoss()
    90.     optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)
    91.  
    92.     epochs = 6
    93.     for epoch in range(epochs):
    94.         model.train()
    95.  
    96.         for image, lables in tqdm(train_loader):
    97.             image = image.to(device)
    98.             lables = lables.to(device)
    99.             running_loss = 0.0
    100.  
    101.             pred = model(image)
    102.             loss = crossLoss(pred, lables)
    103.             running_loss += loss.item()
    104.  
    105.             optimizer.zero_grad()
    106.             loss.backward()
    107.             optimizer.step()
    108.  
    109.         model.eval()
    110.         num_correct = 0
    111.         num_samples = 0
    112.  
    113.         with torch.no_grad():
    114.             for image, lables in test_loader:
    115.                 image = image.to(device)
    116.                 lables = lables.to(device)
    117.  
    118.                 pred = model(image)
    119.  
    120.                 predictions = pred.max(1).indices
    121.                 num_correct += (predictions == lables).sum()
    122.                 num_samples +=predictions.size(0)
    123.  
    124.             acc = (num_correct / num_samples).item()
    125.             model.train()
    126.  
    127.             print('Epoch:{}\t Accuracy:{:.5f}'.format(epoch + 1, acc))
    128.             print('loss : ' + str(running_loss))
    129.  
    130.             txt = open("./test_student.txt", "a").write(
    131.                 (str(epoch + 1) + "    " + str(acc) + "    " + str(running_loss) + '\r\n'))
    132.  
    133.

    训练的效果如下：

    1    0.3483999967575073    1.81593656539917

2    0.4001999795436859    1.9031776189804077

3    0.43629997968673706    1.633713960647583

4    0.4610999822616577    1.5931921005249023

5    0.4650999903678894    1.601379632949829

6    0.47529998421669006    1.547680377960205

    接下来就是蒸馏网络了：

    1. # -*- coding:utf-8 -*-
    2. # @Time : 2022-06-30 18:49
    3. # @Author : DaFuChen
    4. # @File : distillation.py
    5. # @software: PyCharm
    6.  
    7. import torch
    8. import torch.nn as nn
    9. import torch.nn.functional as F
    10. import torchvision
    11. from torchvision.transforms import ToTensor
    12. from tqdm import tqdm
    13. import numpy as np
    14.  
    15.  
    16. # 这里引用不了的一个问题就是------我在student_moudle.py文件那里使用了 if __name__ == '__main__'的语句，导致缩径造成引入失败
    17. # import student_moudle
    18. import teacher_moudle
    19.  
    20.  
    21. if __name__ == "__main__":
    22.  
    23.     file_name_path = "distillation" + ".txt"
    24.     file_name_path = (file_name_path, "w")
    25.  
    26.  
    27.  
    28.     torch.manual_seed(8)
    29.     device = torch.device("cuda" if torch.cuda.is_available()  else "cpu")
    30.     torch.backends.cudnn.benchmark = True
    31.  
    32.     train_set = torchvision.datasets.CIFAR10(
    33.         root="./datac10",
    34.         train=True,
    35.         transform=ToTensor(),
    36.         download=True
    37.     )
    38.  
    39.     test_set = torchvision.datasets.CIFAR10(
    40.         root="./datac10",
    41.         train=False,
    42.         transform=ToTensor(),
    43.         download=True
    44.     )
    45.  
    46.     train_loader = torch.utils.data.DataLoader(
    47.         dataset=train_set,
    48.         batch_size=32,
    49.         shuffle=True
    50.     )
    51.  
    52.     test_loader = torch.utils.data.DataLoader(
    53.         dataset=test_set,
    54.         batch_size=32,
    55.         shuffle=True
    56.     )
    57.  
    58.     # 把一个学生模型在这里进行定义，因为写在其他的py文件的话，会在创建student_model的时候产生训练学生网络的现象
    59.     # 在创建teacher的时候就发生了这样的事情
    60.     class student_moudle(nn.Module):
    61.         def __init__(self, in_channels=1, num_class=10):
    62.             super(student_moudle, self).__init__()
    63.  
    64.             self.conv1 = nn.Conv2d(3, 6, 5)
    65.             self.conv2 = nn.Conv2d(6, 16, 5)
    66.  
    67.             self.pool = nn.MaxPool2d(2, 2)
    68.             self.relu = nn.ReLU()
    69.             self.droput = nn.Dropout(p=0.5)
    70.  
    71.             # 定义全连接层
    72.             self.fc1 = nn.Linear(16 * 5 * 5, 64)
    73.             # 学生的参数是64层，比教师网络的层数少很多，可以使用
    74.             self.fc2 = nn.Linear(64, 32)
    75.             self.fc3 = nn.Linear(32, num_class)
    76.  
    77.         def forward(self, x):
    78.             x = self.pool(self.relu(self.conv1(x)))
    79.             x = self.pool(self.relu(self.conv2(x)))
    80.  
    81.             x = x.view(-1, 16 * 5 * 5)
    82.             x = self.fc1(x)
    83.             x = self.droput(x)
    84.             x = self.relu(x)
    85.  
    86.             x = self.fc2(x)
    87.             x = self.droput(x)
    88.             x = self.relu(x)
    89.  
    90.             x = self.fc3(x)
    91.  
    92.             return x
    93.  
    94.  
    95.  
    96.  
    97.     # 在这里继续定义个teacher_model网络
    98.     class teacherMoudle(nn.Module):
    99.         def __init__(self, in_channels=1, num_class=10):
    100.             super(teacherMoudle, self).__init__()
    101.  
    102.             # 定义卷积
    103.             self.conv1 = nn.Conv2d(3, 6, 5)
    104.             self.conv2 = nn.Conv2d(6, 16, 5)
    105.  
    106.             # 定义池化层
    107.             self.pool = nn.MaxPool2d(2, 2)
    108.  
    109.             # 定义激活函数选择relu()
    110.             self.relu = nn.ReLU()
    111.  
    112.             # 定义droput层
    113.             self.droput = nn.Dropout(p=0.5)
    114.  
    115.             # 定义全连接层
    116.             self.fc1 = nn.Linear(16 * 5 * 5, 1200)
    117.             # 这里需要注意的是教师的模型的隐藏层的参数很对,有1200个
    118.             self.fc2 = nn.Linear(1200, 600)
    119.             self.fc3 = nn.Linear(600, num_class)
    120.  
    121.         def forward(self, x):
    122.             x = self.pool(self.relu(self.conv1(x)))
    123.             x = self.pool(self.relu(self.conv2(x)))
    124.  
    125.             x = x.view(-1, 16 * 5 * 5)
    126.             x = self.fc1(x)
    127.             x = self.droput(x)
    128.             x = self.relu(x)
    129.  
    130.             x = self.fc2(x)
    131.             x = self.droput(x)
    132.             x = self.relu(x)
    133.  
    134.             x = self.fc3(x)
    135.  
    136.             return x
    137.  
    138.  
    139.     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    140.  
    141.     # 先准备已经接受训练的教师模型
    142.     teacher_moudle.model.eval()
    143.  
    144.     # 准备新的学生模型
    145.     # model = student_moudle.student_moudle()
    146.     model = student_moudle()
    147.     model = model.to(device)
    148.  
    149.     # 教师网络需要进行这样一个操作在后面才能使用这个网络进行调用预测image, 因为后面image把它置与GPU中，这里需要把模型也放到GPU中
    150.     teachermoudle = teacherMoudle()
    151.     teachermoudle = teachermoudle.to(device)
    152.  
    153.     model.train()
    154.  
    155.     # 设置需要进行蒸馏的温度
    156.     temp = 7
    157.  
    158.     # 设置蒸馏学习的损失函数 使用的还是交叉熵损失函数
    159.     hard_loss = nn.CrossEntropyLoss()
    160.  
    161.     # 设置学习的损失值的权重
    162.     alpha = 0.3
    163.  
    164.     # 使用一个soft_loss
    165.     soft_loss = nn.KLDivLoss(reduction='batchmean')
    166.  
    167.     # 设置一个优化器
    168.     optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
    169.  
    170.     epochs = 6
    171.     for epoch in range(epochs):
    172.         # 训练集上训练模型的权重
    173.         for image, lables in tqdm(train_loader):
    174.             running_loss = 0
    175.             image = image.to(device)
    176.             lables = lables.to(device)
    177.  
    178.             with torch.no_grad():
    179.                 teacher_preds = teachermoudle(image)
    180.                 # print(teacher_preds)
    181.                 # teacher_preds = teacher_preds.numpy()
    182.             # 学生的模型
    183.             student_preds = model(image)
    184.  
    185.             # 计算hard_loss
    186.             student_loss = hard_loss(student_preds, lables)
    187.  
    188.             # 计算蒸馏后的预测结果soft_loss
    189.             ditillation_loss = soft_loss(
    190.                 F.softmax(student_preds/temp, dim=1),
    191.                 F.softmax(teacher_preds/temp, dim=1)
    192.             )
    193.  
    194.  
    195.             # 将hard_loss和soft_loss加权求和  更新loss值
    196.             loss = alpha*student_loss+(1-alpha)*ditillation_loss
    197.             running_loss += loss.item()
    198.  
    199.             # 反向传播 优化权重
    200.             optimizer.zero_grad()
    201.             loss.backward()
    202.             optimizer.step()
    203.  
    204.         # 测试集上评估模型性能
    205.         model.eval()
    206.  
    207.  
    208.         # model.eval()
    209.         num_correct = 0
    210.         num_samples = 0
    211.  
    212.         with torch.no_grad():
    213.             for image2, lables2 in test_loader:
    214.                 image2 = image2.to(device)
    215.                 lables2 = lables2.to(device)
    216.  
    217.                 preds = model(image2)
    218.                 predictions = preds.max(1).indices
    219.                 num_correct += (predictions == lables2).sum()
    220.                 num_samples += predictions.size(0)
    221.             acc = (num_correct/num_samples).item()
    222.         model.train()
    223.  
    224.         print('Epoch:{}\t Accuracy:{:.5f}'.format(epoch + 1, acc))
    225.         print('loss : ' + str(running_loss))
    226.  
    227.         txt = open("./distillation.txt", "a").write(
    228.             (str(epoch + 1) + "    " + str(acc) + "    " + str(running_loss) + '\r\n'))
    229.  
    230.  
    231.  
    232.

    训练效果如下：

    1    0.3822000026702881    -1.1664808988571167

2    0.41040000319480896    -1.1804652214050293

3    0.46219998598098755    -1.2277178764343262

4    0.4693000018596649    -1.302511215209961

5    0.4846999943256378    -1.2454404830932617

6    0.4932999908924103    -1.27120840549469

    本人还是大二的，对深度学习不是很深入了解，如果上面有错误，欢迎指正，请别互联网暴力哇，发这个博客纯属是为了给大伙一起学习新知识

  • 相关阅读:
    R语言 | 绘制带P值的差异柱状图
    EasyExcel表头校验方法
    【初阶数据结构】——堆排序和TopK问题
    877. 扩展欧几里得算法
    js字符串处理
    数据结构题型11-顺序队列
    如何在Python爬虫中使用IP代理以避免反爬虫机制
    力扣刷题19-删除链表的倒数第N个节点
    JDBC工具类
    Intel-Hex , Motorola S-Record 格式详细解析
  • 原文地址：https://blog.csdn.net/blockshowtouse/article/details/125551415