• 周志华《机器学习》第三章课后习题

    目录

    3.1 试析在什么情形下式(3.2) 中不必考虑偏置项 b.

    3.2、试证明，对于参数w,对率回归的目标函数(3.18)是非凸的,但其对数似然函数(3.27)是凸的.

     3.3、编程实现对率回归,并给出西瓜数据集3.0α上的结果.

    3.4 选择两个 UCI 数据集，比较 10 折交叉验证法和留一法所估计出的对率回归的错误率。

    3.5 编辑实现线性判别分析，并给出西瓜数据集 3.0α 上的结果.

    3.1 试析在什么情形下式(3.2) 中不必考虑偏置项 b.

    请添加图片描述


    ①b与输入毫无关系，如果没有b，y‘=wx必须经过原点
    ②当两个线性模型相减时，消除了b。可用训练集中每个样本都减去第一个样本，然后对新的样本做线性回归，不用考虑偏置项b。

    3.2、试证明，对于参数w,对率回归的目标函数(3.18)是非凸的,但其对数似然函数(3.27)是凸的.

     3.27

     

     3.3、编程实现对率回归,并给出西瓜数据集3.0α上的结果.

    数据集：

    3.3.py

    1.  
    2. # -*- coding: utf-8 -*
    3.  
    4. '''
    5. data importion
    6. '''
    7. import numpy as np  # for matrix calculation
    8. import matplotlib.pyplot as plt
    9.  
    10. # load the CSV file as a numpy matrix
    11. # 将CSV文件加载为numpy矩阵
    12. dataset = np.loadtxt('watermelon3_0_Ch.csv', delimiter=",")
    13.  
    14. # separate the data from the target attributes
    15. # 将数据与目标属性分离
    16. X = dataset[:, 1:3]
    17. y = dataset[:, 3]
    18.  
    19. m, n = np.shape(X)
    20.  
    21. # draw scatter diagram to show the raw data
    22. #绘制出数据点
    23. f1 = plt.figure(1)
    24. plt.title('watermelon_3a')
    25. plt.xlabel('density')
    26. plt.ylabel('ratio_sugar')
    27. plt.scatter(X[y == 0, 0], X[y == 0, 1], marker='o', color='k', s=100, label='bad')
    28. plt.scatter(X[y == 1, 0], X[y == 1, 1], marker='o', color='g', s=100, label='good')
    29. plt.legend(loc='upper right')
    30. # plt.show()
    31.  
    32. ''' 
    33. using sklearn lib for logistic regression
    34. 使用sklearn库进行逻辑回归
    35. '''
    36. from sklearn import metrics
    37. from sklearn import model_selection
    38. from sklearn.linear_model import LogisticRegression
    39. import matplotlib.pylab as pl
    40.  
    41. # generalization of test and train set
    42. # 先划分训练集和测试集，采用sklearn.model_selection.train_test_split()实现
    43. X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.5, random_state=0)
    44.  
    45. # model training
    46. # 采用sklearn.linear_model.LogisticRegression，基于训练集直接拟合出逻辑回归模型，然后在测试集上评估模型（查看混淆矩阵和F1值）
    47.  
    48. log_model = LogisticRegression()  # using log-regression lib model
    49. log_model.fit(X_train, y_train)  # fitting
    50.  
    51. # model validation 模型确认
    52. y_pred = log_model.predict(X_test)
    53.  
    54. # summarize the fit of the model  总结模型的拟合情况
    55. print(metrics.confusion_matrix(y_test, y_pred))
    56. print(metrics.classification_report(y_test, y_pred))
    57.  
    58. precision, recall, thresholds = metrics.precision_recall_curve(y_test, y_pred)
    59.  
    60. # show decision boundary in plt  在PLT中显示决策边界
    61. # X - some data in 2dimensional np.array   X -二维np.array中的一些数据
    62. f2 = plt.figure(2)
    63. h = 0.001
    64. x0_min, x0_max = X[:, 0].min() - 0.1, X[:, 0].max() + 0.1
    65. x1_min, x1_max = X[:, 1].min() - 0.1, X[:, 1].max() + 0.1
    66. x0, x1 = np.meshgrid(np.arange(x0_min, x0_max, h),
    67.                      np.arange(x1_min, x1_max, h))
    68.  
    69. # here "model" is your model's prediction (classification) function
    70. # 这里的“模型”是模型的预测(分类)函数
    71. z = log_model.predict(np.c_[x0.ravel(), x1.ravel()])
    72.  
    73. # Put the result into a color plot 把结果放入颜色图中
    74. z = z.reshape(x0.shape)
    75. # 采用matplotlib.contourf绘制的决策区域和边界，可以看出对率回归分类器还是成功的分出了绝大多数类：
    76. plt.contourf(x0, x1, z, cmap=pl.cm.Paired)
    77.  
    78. # Plot also the training pointsplt.title('watermelon_3a')
    79. plt.title('watermelon_3a')
    80. plt.xlabel('density')
    81. plt.ylabel('ratio_sugar')
    82. plt.scatter(X[y == 0, 0], X[y == 0, 1], marker='o', color='k', s=100, label='bad')
    83. plt.scatter(X[y == 1, 0], X[y == 1, 1], marker='o', color='g', s=100, label='good')
    84. # plt.show()
    85.  
    86. '''
    87. coding to implement logistic regression
    88. 编码以实现逻辑回归
    89. '''
    90. from sklearn import model_selection
    91.  
    92. import self_def
    93.  
    94. # X_train, X_test, y_train, y_test
    95. np.ones(n)
    96. m, n = np.shape(X)
    97. X_ex = np.c_[X, np.ones(m)]  # extend the variable matrix to [x, 1]
    98. X_train, X_test, y_train, y_test = model_selection.train_test_split(X_ex, y, test_size=0.5, random_state=0)
    99.  
    100. # using gradDescent to get the optimal parameter beta = [w, b] in page-59
    101. beta = self_def.gradDscent_2(X_train, y_train)
    102.  
    103. # prediction, beta mapping to the model
    104. y_pred = self_def.predict(X_test, beta)
    105.  
    106. m_test = np.shape(X_test)[0]
    107. # calculation of confusion_matrix and prediction accuracy
    108. # #混淆矩阵的计算和预测精度
    109. cfmat = np.zeros((2, 2))
    110. for i in range(m_test):
    111.     if y_pred[i] == y_test[i] == 0:
    112.         cfmat[0, 0] += 1
    113.     elif y_pred[i] == y_test[i] == 1:
    114.         cfmat[1, 1] += 1
    115.     elif y_pred[i] == 0:
    116.         cfmat[1, 0] += 1
    117.     elif y_pred[i] == 1:
    118.         cfmat[0, 1] += 1
    119.  
    120. print(cfmat)
    121.  
    122.  
    123.  
    124.  
    125.  
    126.  
    127.

    self_def.py  是 需要调用的函数

    1. import numpy as np
    2.  
    3.  
    4. def likelihood_sub(x, y, beta):
    5.     '''
    6.     @param X: one sample variables
    7.     @param y: one sample label
    8.     @param beta: the parameter vector in 3.27
    9.     @return: the sub_log-likelihood of 3.27
    10.     3.27式子的变成对象
    11.     '''
    12.     return -y * np.dot(beta, x.T) + np.math.log(1 + np.math.exp(np.dot(beta, x.T)))
    13.  
    14.  
    15. def likelihood(X, y, beta):
    16.     '''
    17.     @param X: the sample variables matrix
    18.     @param y: the sample label matrix
    19.     @param beta: the parameter vector in 3.27
    20.     @return: the log-likelihood of 3.27
    21.     '''
    22.     sum = 0
    23.     m, n = np.shape(X)
    24.  
    25.     for i in range(m):
    26.         sum += likelihood_sub(X[i], y[i], beta)
    27.  
    28.     return sum
    29.  
    30.  
    31. def partial_derivative(X, y, beta):  # refer to 3.30 on book page 60  请参阅第60页的3.30
    32.     '''
    33.     @param X: the sample variables matrix
    34.     @param y: the sample label matrix
    35.     @param X:样本变量矩阵
    36.      @param y:样本标签矩阵
    37.     @param beta: the parameter vector in 3.27
    38.     @return: the partial derivative of beta [j]
    39.     '''
    40.  
    41.     m, n = np.shape(X)
    42.     pd = np.zeros(n)
    43.  
    44.     for i in range(m):
    45.         tmp = y[i] - sigmoid(X[i], beta)
    46.         for j in range(n):
    47.             pd[j] += X[i][j] * (tmp)
    48.     return pd
    49.  
    50.  
    51. def gradDscent_1(X, y):  # implementation of fundational gradDscent algorithms 基本梯度算法的实现
    52.     '''
    53.     @param X: X is the variable matrix
    54.     @param y: y is the label array
    55.     @return: the best parameter estimate of 3.27
    56.     然后基于训练集（注意x->[x,1]），给出基于3.27似然函数的定步长梯度下降法，降低损失，注意这里的偏梯度实现技巧：
    57.     '''
    58.     import matplotlib.pyplot as plt
    59.  
    60.     h = 0.1  # step length of iterator  迭代器的步长
    61.     max_times = 500  # give the iterative times limit  给出迭代次数的极限
    62.     m, n = np.shape(X)
    63.  
    64.     b = np.zeros((n, max_times))  # for show convergence curve of parameter 表示参数的收敛曲线
    65.     beta = np.zeros(n)  # parameter and initial  参数和初始
    66.     delta_beta = np.ones(n) * h
    67.     llh = 0
    68.     llh_temp = 0
    69.  
    70.     for i in range(max_times):
    71.         beta_temp = beta.copy()
    72.  
    73.         for j in range(n):
    74.             # for partial derivative  偏导数
    75.             beta[j] += delta_beta[j]
    76.             llh_tmp = likelihood(X, y, beta)
    77.             delta_beta[j] = -h * (llh_tmp - llh) / delta_beta[j]
    78.  
    79.             b[j, i] = beta[j]
    80.  
    81.             beta[j] = beta_temp[j]
    82.  
    83.         beta += delta_beta
    84.         llh = likelihood(X, y, beta)
    85.  
    86.     t = np.arange(max_times)
    87.  
    88.     f2 = plt.figure(3)
    89.  
    90.     p1 = plt.subplot(311)
    91.     p1.plot(t, b[0])
    92.     plt.ylabel('w1')
    93.  
    94.     p2 = plt.subplot(312)
    95.     p2.plot(t, b[1])
    96.     plt.ylabel('w2')
    97.  
    98.     p3 = plt.subplot(313)
    99.     p3.plot(t, b[2])
    100.     plt.ylabel('b')
    101.  
    102.     plt.show()
    103.     return beta
    104. '''
    105. 采用随机梯度下降法来优化：上面采用的是全局定步长梯度下降法（称之为批量梯度下降），
    106. 这种方法在可能会面临收敛过慢和收敛曲线波动情况的同时，每次迭代需要全局计算，
    107. 计算量随数据量增大而急剧增大。所以尝试采用随机梯度下降来改善参数迭代寻优过程。
    108. '''
    109.  
    110. def gradDscent_2(X, y):  # implementation of stochastic gradDscent algorithms  随机梯度算法的实现
    111.     '''
    112.     @param X: X is the variable matrix
    113.     @param y: y is the label array
    114.     @return: the best parameter estimate of 3.27
    115.     随机梯度下降法的核心思想是增量学习：一次只用一个新样本来更新回归系数，从而形成在线流式处理。
    117.      同时为了加快收敛，采用变步长的策略，h随着迭代次数逐渐减小。
    118.     '''
    119.     import matplotlib.pyplot as plt
    120.  
    121.     m, n = np.shape(X)
    122.     h = 0.5  # step length of iterator and initial
    123.     beta = np.zeros(n)  # parameter and initial
    124.     delta_beta = np.ones(n) * h
    125.     llh = 0
    126.     llh_temp = 0
    127.     b = np.zeros((n, m))  # for show convergence curve of parameter
    128.  
    129.     for i in range(m):
    130.         beta_temp = beta.copy()
    131.  
    132.         for j in range(n):
    133.             # for partial derivative
    134.             h = 0.5 * 1 / (1 + i + j)  # change step length of iterator
    135.             beta[j] += delta_beta[j]
    136.  
    137.             b[j, i] = beta[j]
    138.  
    139.             llh_tmp = likelihood_sub(X[i], y[i], beta)
    140.             delta_beta[j] = -h * (llh_tmp - llh) / delta_beta[j]
    141.  
    142.             beta[j] = beta_temp[j]
    143.  
    144.         beta += delta_beta
    145.         llh = likelihood_sub(X[i], y[i], beta)
    146.  
    147.     t = np.arange(m)
    148.  
    149.     f2 = plt.figure(3)
    150.  
    151.     p1 = plt.subplot(311)
    152.     p1.plot(t, b[0])
    153.     plt.ylabel('w1')
    154.  
    155.     p2 = plt.subplot(312)
    156.     p2.plot(t, b[1])
    157.     plt.ylabel('w2')
    158.  
    159.     p3 = plt.subplot(313)
    160.     p3.plot(t, b[2])
    161.     plt.ylabel('b')
    162.  
    163.     plt.show()
    164.  
    165.     return beta
    166.  
    167. #sigmoid函数
    168. def sigmoid(x, beta):
    169.     '''
    170.     @param x: is the predict variable
    171.     @param beta: is the parameter
    172.     @return: the sigmoid function value
    173.     
    174.     '''
    175.     return 1.0 / (1 + np.math.exp(- np.dot(beta, x.T)))
    176.  
    177.  
    178. def predict(X, beta):
    179.     '''
    180.     prediction the class lable using sigmoid  使用sigmoid预测类标签
    181.     @param X: data sample form like [x, 1]   数据样本形式如[x, 1]
    182.     @param beta: the parameter of sigmoid form like [w, b]  形如[w, b]的参数
    183.     @return: the class lable array  类标签数组
    184.     '''
    185.     m, n = np.shape(X)
    186.     y = np.zeros(m)
    187.  
    188.     for i in range(m):
    189.         if sigmoid(X[i], beta) > 0.5: y[i] = 1;
    190.     return y
    191.  
    192.     return

    3.4 选择两个 UCI 数据集，比较 10 折交叉验证法和留一法所估计出的对率回归的错误率。

    参考代码： han1057578619/MachineLearning_Zhouzhihua_ProblemSets

    3.5 编辑实现线性判别分析，并给出西瓜数据集 3.0α 上的结果.

     3.5.py

    1. import numpy as np
    2. import pandas as pd
    3. from matplotlib import pyplot as plt
    4.  
    5.  
    6. class LDA(object):
    7.     # 绘图，求出均值向量，根据公式3.34和3.39求出类内散度矩阵和类间散度矩阵
    8.  
    9.     def fit(self, X_, y_, plot_=False):
    10.         pos = y_ == 1
    11.         neg = y_ == 0
    12.         X0 = X_[neg]
    13.         X1 = X_[pos]
    14.         # 均值向量，(1, 2)
    15.  
    16.         u0 = X0.mean(0, keepdims=True)  # (1, n)
    17.         u1 = X1.mean(0, keepdims=True)
    18.  
    19.          # 类内散度矩阵，公式3.33，(2, 2)
    20.         sw = np.dot((X0 - u0).T, (X0 - u0)) + np.dot((X1 - u1).T, (X1 - u1))
    21.         # 类间散度矩阵，公式3.37，(1, 2)
    22.         w = np.dot(np.linalg.inv(sw), (u0 - u1).T).reshape(1, -1)
    23.  
    24.         if plot_:
    25.             fig, ax = plt.subplots()
    26.             ax.spines['right'].set_color('none')
    27.             ax.spines['top'].set_color('none')
    28.             ax.spines['left'].set_position(('data', 0))
    29.             ax.spines['bottom'].set_position(('data', 0))
    30.  
    31.             plt.scatter(X1[:, 0], X1[:, 1], c='k', marker='o', label='good')
    32.             plt.scatter(X0[:, 0], X0[:, 1], c='r', marker='x', label='bad')
    33.  
    34.             plt.xlabel('密度', labelpad=1)
    35.             plt.ylabel('含糖量')
    36.             plt.legend(loc='upper right')
    37.  
    38.             x_tmp = np.linspace(-0.05, 0.15)
    39.             y_tmp = x_tmp * w[0, 1] / w[0, 0]
    40.             plt.plot(x_tmp, y_tmp, '#808080', linewidth=1)
    41.  
    42.             wu = w / np.linalg.norm(w)
    43.  
    44.             # 正负样板店
    45.             X0_project = np.dot(X0, np.dot(wu.T, wu))
    46.             plt.scatter(X0_project[:, 0], X0_project[:, 1], c='r', s=15)
    47.             for i in range(X0.shape[0]):
    48.                 plt.plot([X0[i, 0], X0_project[i, 0]], [X0[i, 1], X0_project[i, 1]], '--r', linewidth=1)
    49.  
    50.             X1_project = np.dot(X1, np.dot(wu.T, wu))
    51.             plt.scatter(X1_project[:, 0], X1_project[:, 1], c='k', s=15)
    52.             for i in range(X1.shape[0]):
    53.                 plt.plot([X1[i, 0], X1_project[i, 0]], [X1[i, 1], X1_project[i, 1]], '--k', linewidth=1)
    54.  
    55.             # 中心点的投影
    56.             u0_project = np.dot(u0, np.dot(wu.T, wu))
    57.             plt.scatter(u0_project[:, 0], u0_project[:, 1], c='#FF4500', s=60)
    58.             u1_project = np.dot(u1, np.dot(wu.T, wu))
    59.             plt.scatter(u1_project[:, 0], u1_project[:, 1], c='#696969', s=60)
    60.             # 均值向量的投影点
    61.  
    62.             ax.annotate(r'u0 投影点',
    63.                         xy=(u0_project[:, 0], u0_project[:, 1]),
    64.                         xytext=(u0_project[:, 0] - 0.2, u0_project[:, 1] - 0.1),
    65.                         size=13,
    66.                         va="center", ha="left",
    67.                         arrowprops=dict(arrowstyle="->",
    68.                                         color="k",
    69.                                         )
    70.                         )
    71.  
    72.             ax.annotate(r'u1 投影点',
    73.                         xy=(u1_project[:, 0], u1_project[:, 1]),
    74.                         xytext=(u1_project[:, 0] - 0.1, u1_project[:, 1] + 0.1),
    75.                         size=13,
    76.                         va="center", ha="left",
    77.                         arrowprops=dict(arrowstyle="->",
    78.                                         color="k",
    79.                                         )
    80.                         )
    81.             plt.axis("equal")  # 两坐标轴的单位刻度长度保存一致
    82.             plt.show()
    83.  
    84.         self.w = w
    85.         self.u0 = u0
    86.         self.u1 = u1
    87.         return self
    88.  
    89.     def predict(self, X):
    90.         project = np.dot(X, self.w.T)
    91.  
    92.         wu0 = np.dot(self.w, self.u0.T)
    93.         wu1 = np.dot(self.w, self.u1.T)
    94.  
    95.         return (np.abs(project - wu1) < np.abs(project - wu0)).astype(int)
    96.  
    97.  
    98. if __name__ == '__main__':
    99.     data_path = r'watermelon3_0_Ch.csv'
    100.  
    101.     data = pd.read_csv(data_path).values
    102.  
    103.     X = data[:, 1:3].astype(float)
    104.     y = data[:, 3]
    105.  
    106.     y[y == '是'] = 1
    107.     y[y == '否'] = 0
    108.     y = y.astype(int)
    109.  
    110.     lda = LDA()
    111.     lda.fit(X, y, plot_=True)
    112.     print(lda.predict(X))  # 和逻辑回归的结果一致
    113.     print(y)
    114.

     

    想要代码与数据资源的，可以加我微信好友

     

    参考的博客：

    (4条消息) 周志华《机器学习》课后习题第三章解答：Ch3.3 - 编程实现对率回归_zhangriqi的博客-CSDN博客

    周志华《机器学习》课后习题（第三章）：线性模型-阿里云开发者社区 (aliyun.com) 

  • 相关阅读:
    linux 命令模式下我如何移动光标呢
    383.赎金信
    CRM系统如何帮助企业数字化转型？分为几个阶段？
    一文详解训练LLM流程
    21. [Python GUI] PyQt5中的模型与视图框架-抽象模型基类QAbstractItemModel与自定义模型
    Java_线程的概念和线程的创建的方法
    2021年软件测试面试题大全
    博士论文答辩流程
    vue3弹性布局 类似九宫格排列（多选选项）
    在面试提问环节应该问那些内容
  • 原文地址：https://blog.csdn.net/qq_40694323/article/details/125410908