• 机器学习：集成学习（Python）

    一、Adaboost算法

    1.1 Adaboost分类算法

    adaboost_discrete_c.py

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_C import DecisionTreeClassifier
    4.  
    5.  
    6. class AdaBoostClassifier:
    7.     """
    8.     adaboost分类算法：既可以做二分类、也可以做多分类，取决于基本分类器
    9.     1. 同质学习器：非列表形式，按同种类型的基学习器构造
    10.     2. 异质学习器：列表传递[logisticsregression, svm, cart, ...]
    11.     """
    12.     def __init__(self, base_estimator=None, n_estimators=10, learning_rate=1.0):
    13.         """
    14.         :param base_estimator: 基学习器
    15.         :param n_estimators: 基学习器的个数T
    16.         :param learning_rate: 学习率，降低后续训练的基分类器的权重，避免过拟合
    17.         """
    18.         self.base_estimator = base_estimator
    19.         self.n_estimators = n_estimators
    20.         self.learning_rate = learning_rate
    21.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    22.         if self.base_estimator is None:
    23.             self.base_estimator = DecisionTreeClassifier(max_depth=2)
    24.         if type(base_estimator) != list:
    25.             # 同质（同种类型）的分类器，深拷贝
    26.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    27.                                    for _ in range(self.n_estimators)]
    28.         else:
    29.             # 异质（不同种类型）的分类器
    30.             self.n_estimators = len(self.base_estimator)
    31.         self.estimator_weights = []  # 每个基学习器的权重系数
    32.  
    33.     def fit(self, x_train, y_train):
    34.         """
    35.         训练AdaBoost每个基学习器，计算权重分布，每个基学习器的误差率和权重系数α，
    36.         :param x_train: 训练集，二维数组：m * k
    37.         :param y_train: 目标集
    38.         :return:
    39.         """
    40.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    41.         n_samples, n_class = x_train.shape[0], len(set(y_train))  # 样本量，类别数
    42.         sample_weight = np.ones(n_samples)  # 为适应自写的基学习器，设置样本均匀权重为1.0
    43.         # 针对每一个基学习器，根据带有权重分布的训练集训练基学习器，计算相关参数
    44.         for idx in range(self.n_estimators):
    45.             # 1. 使用具有权重分布的Dm的训练数据集学习，并预测
    46.             self.base_estimator[idx].fit(x_train, y_train, sample_weight=sample_weight)
    47.             # 只关心分类错误的，如果分类错误，则为0，正确则为1
    48.             y_hat_0 = (self.base_estimator[idx].predict(x_train) == y_train).astype(int)
    49.             # 2. 计算分类误差率
    50.             error_rate = sample_weight.dot(1.0 - y_hat_0) / n_samples
    51.             if error_rate > 0.5:
    52.                 self.estimator_weights.append(0)  # 当前基分类器不起作用
    53.                 continue
    54.             # 3. 计算基分类器的权重系数，考虑溢出
    55.             alpha_rate = 0.5 * np.log((1 - error_rate) / error_rate + 1e-8) + np.log(n_class - 1)
    56.             alpha_rate = min(10.0, alpha_rate)  # 避免权重系数过大
    57.             self.estimator_weights.append(alpha_rate)
    58.             # 4. 更新样本权重，为了适应多分类，yi*Gm(xi)计算np.power(-1.0, 1 - y_hat_0)
    59.             sample_weight *= np.exp(-1.0 * alpha_rate * np.power(-1.0, 1 - y_hat_0))
    60.             sample_weight = sample_weight / np.sum(sample_weight) * n_samples
    61.         # 5. 更新estimator的权重系数，按照学习率
    62.         for i in range(self.n_estimators):
    63.             self.estimator_weights[i] *= np.power(self.learning_rate, i)
    64.  
    65.     def predict_proba(self, x_test):
    66.         """
    67.         预测测试样本所属类别概率，软投票
    68.         :param x_test: 测试样本集
    69.         :return:
    70.         """
    71.         x_test = np.asarray(x_test)
    72.         # 按照加法模型，线性组合基学习器
    73.         # 每个测试样本，每个基学习器预测概率（10，[（0.68， 0.32），（0.55， 0.45）]...）
    74.         y_hat_prob = np.sum([self.base_estimator[i].predict_proba(x_test) *
    75.                              self.estimator_weights[i] for i in range(self.n_estimators)], axis=0)
    76.         return y_hat_prob / y_hat_prob.sum(axis=1, keepdims=True)
    77.  
    78.     def predict(self, x_test):
    79.         """
    80.         预测测试样本所属类别
    81.         :param x_test: 测试样本集
    82.         :return:
    83.         """
    84.         return np.argmax(self.predict_proba(x_test), axis=1)
    85.

    1.2 Adaboost回归算法

    adaboost_regressor.py

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4.  
    5.  
    6. class AdaBoostRegressior:
    7.     """
    8.     adaboost回归算法：结合（集成）策略：加权中位数、预测值的平均加权
    9.     1. 同质学习器，异质学习器
    10.     2. 回归误差率依赖于相对误差：平方误差、线性误差、指数误差
    11.     """
    12.     def __init__(self, base_estimator=None, n_estimators=10, learning_rate=1.0,
    13.                  loss="square", comb_strategy="weight_median"):
    14.         """
    15.         :param base_estimator: 基学习器
    16.         :param n_estimators: 基学习器的个数T
    17.         :param learning_rate: 学习率，降低后续训练的基分类器的权重，避免过拟合
    18.         :param loss: 损失函数：linear、square、exp
    19.         :param comb_strategy: weight_median、weight_mean
    20.         """
    21.         self.base_estimator = base_estimator
    22.         self.n_estimators = n_estimators
    23.         self.learning_rate = learning_rate
    24.         self.loss = loss  # 相对误差的损失函数
    25.         self.comb_strategy = comb_strategy  # 结合策略
    26.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    27.         if self.base_estimator is None:
    28.             self.base_estimator = DecisionTreeRegression(max_depth=2)
    29.         if type(base_estimator) != list:
    30.             # 同质（同种类型）的分类器，深拷贝
    31.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    32.                                    for _ in range(self.n_estimators)]
    33.         else:
    34.             # 异质（不同种类型）的分类器
    35.             self.n_estimators = len(self.base_estimator)
    36.         self.estimator_weights = []  # 每个基学习器的权重系数
    37.  
    38.     def _cal_loss(self, y_true, y_hat):
    39.         """
    40.         根据损失函数计算相对误差
    41.         :param y_true: 真值
    42.         :param y_hat: 预测值
    43.         :return:
    44.         """
    45.         errors = np.abs(y_true - y_hat)  # 绝对值误差
    46.         if self.loss.lower() == "linear":  # 线性
    47.             return errors / np.max(errors)
    48.         elif self.loss.lower() == "square":  # 平方
    49.             errors_s = (y_true - y_hat) ** 2
    50.             return errors_s / np.max(errors) ** 2
    51.         elif self.loss.lower() == "exp":  # 指数
    52.             return 1 - np.exp(-errors / np.max(errors))
    53.         else:
    54.             raise ValueError("仅支持linear、square和exp...")
    55.  
    56.     def fit(self, x_train, y_train):
    57.         """
    58.         Adaboost回归算法，T个基学习器的训练：
    59.         1. 基学习器基于权重分布Dt的训练集训练
    60.         2. 计算最大绝对误差、相对误差、回归误差率
    61.         3. 计算当前ht的置信度
    62.         4. 更新下一轮的权重分布
    63.         :param x_train:
    64.         :param y_train:
    65.         :return:
    66.         """
    67.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    68.         n_samples, n_class = x_train.shape[0], len(set(y_train))  # 样本量，类别数
    69.         sample_weight = np.ones(n_samples)  # 为适应自写的基学习器，设置样本均匀权重为1.0
    70.         for idx in range(self.n_estimators):
    71.             # 1. 基学习器基于权重分布Dt的训练集训练以及预测
    72.             self.base_estimator[idx].fit(x_train, y_train, sample_weight=sample_weight)
    73.             y_hat = self.base_estimator[idx].predict(x_train)  # 当前训练集的预测值
    74.             # 2. 计算最大绝对误差、相对误差、回归误差率
    75.             errors = self._cal_loss(y_train, y_hat)  # 相对误差
    76.             error_rate = np.dot(errors, sample_weight / n_samples)  # 回归误差率
    77.             # 3. 计算当前ht的置信度，基学习器的权重参数
    78.             alpha_rate = error_rate / (1 - error_rate)
    79.             self.estimator_weights.append(alpha_rate)
    80.             # 4. 更新下一轮的权重分布
    81.             sample_weight *= np.power(alpha_rate, 1 - errors)
    82.             sample_weight = sample_weight / np.sum(sample_weight) * n_samples
    83.         # 5. 计算基学习器的权重系数以及考虑学习率
    84.         self.estimator_weights = np.log(1 / np.asarray(self.estimator_weights))
    85.         for i in range(self.n_estimators):
    86.             self.estimator_weights[i] *= np.power(self.learning_rate, i)
    87.  
    88.     def predict(self, x_test):
    89.         """
    90.         Adaboost回归算法预测，按照加权中位数以及加权平均两种结合策略
    91.         :param x_test: 测试样本集
    92.         :return:
    93.         """
    94.         x_test = np.asarray(x_test)
    95.         if self.comb_strategy == "weight_mean":  # 加权平均
    96.             self.estimator_weights /= np.sum(self.estimator_weights)
    97.             # n * T
    98.             y_hat_mat = np.array([self.estimator_weights[i] *
    99.                                   self.base_estimator[i].predict(x_test)
    100.                                   for i in range(self.n_estimators)])
    101.             # print(y_hat_mat.shape) (10, 5160)
    102.             return np.sum(y_hat_mat, axis=0)
    103.         elif self.comb_strategy == "weight_median":  # 加权中位数
    104.             # T个基学习器的预测结果构成一个二维数组（10, 5160）
    105.             y_hat_mat = np.array([self.estimator_weights[i] *
    106.                                   self.base_estimator[i].predict(x_test)
    107.                                   for i in range(self.n_estimators)]).T
    108.             sorted_idx = np.argsort(y_hat_mat, axis=1)  # 二维数组
    109.             # 按照每个样本预测值的升序排列序号，排序权重系数，然后累加计算
    110.             weight_cdf = np.cumsum(self.estimator_weights[sorted_idx], axis=1)
    111.             # 选择最小的t，如下代码产生二维bool数组
    112.             median_or_above = weight_cdf >= 0.5 * weight_cdf[:, -1][:, np.newaxis]
    113.             # print(median_idx)
    114.             median_idx = np.argmax(median_or_above, axis=1)  # 返回每个样本的t索引值
    115.             median_estimators = sorted_idx[np.arange(x_test.shape[0]), median_idx]
    116.             return y_hat_mat[np.arange(x_test.shape[0]), median_estimators]
    117.  
    118.

    1.3 SAMME算法

    samme_r_muti_classifier.py 

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_C import DecisionTreeClassifier
    4.  
    5.  
    6. class SAMMERClassifier:
    7.     """
    8.     SAMME.R算法是将SAMME拓展到连续数值型的范畴。
    9.     基学习器的输出为连续型，一般为类别概率的预测值。
    10.     """
    11.     def __init__(self, base_estimator=None, n_estimators=10):
    12.         """
    13.         :param base_estimator: 基学习器
    14.         :param n_estimators: 基学习器的个数T
    15.         """
    16.         self.base_estimator = base_estimator
    17.         self.n_estimators = n_estimators
    18.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    19.         if self.base_estimator is None:
    20.             self.base_estimator = DecisionTreeClassifier(max_depth=2)
    21.         if type(base_estimator) != list:
    22.             # 同质（同种类型）的分类器，深拷贝
    23.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    24.                                    for _ in range(self.n_estimators)]
    25.         else:
    26.             # 异质（不同种类型）的分类器
    27.             self.n_estimators = len(self.base_estimator)
    28.         self.estimator_weights = []  # 每个基学习器的权重系数
    29.         self.n_samples, self.n_class = None, None  # 样本量和类别数
    30.  
    31.     def _target_encoding(self, y_train):
    32.         """
    33.         对目标值进行编码
    34.         :param y_train: 训练目标集
    35.         :return:
    36.         """
    37.         self.n_samples, self.n_class = len(y_train), len(set(y_train))
    38.         target = -1 / (self.n_class - 1) * np.ones((self.n_samples, self.n_class))
    39.         for i in range(self.n_samples):
    40.             target[i, y_train[i]] = 1  # 对应该样本的类别所在编码中的列改为1
    41.         return target
    42.  
    43.     def fit(self, x_train, y_train):
    44.         """
    45.         训练SAMME.Rt每个基学习器，根据预测类别概率计算权重分布
    46.         :param x_train: 训练集，二维数组：m * k
    47.         :param y_train: 目标集
    48.         :return:
    49.         """
    50.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    51.         target = self._target_encoding(y_train)  # 编码
    52.         sample_weight = np.ones(self.n_samples)  # 为适应自写的基学习器，设置样本均匀权重为1.0
    53.         # 针对每一个基学习器，根据带有权重分布的训练集训练基学习器，计算相关参数
    54.         c = (self.n_class - 1) / self.n_class
    55.         for idx in range(self.n_estimators):
    56.             # 1. 使用具有权重分布的Dm的训练数据集学习，并预测
    57.             self.base_estimator[idx].fit(x_train, y_train, sample_weight=sample_weight)
    58.             # 根据训练的基学习器，获得其样本的预测类别概率
    59.             pred_p = self.base_estimator[idx].predict_proba(x_train)
    60.             # 针对预测概率，小于eps的值替换为eps，避免log函数溢出
    61.             np.clip(pred_p, np.finfo(pred_p.dtype).eps, None, out=pred_p)
    62.             # 2. 更新样本权重
    63.             sample_weight *= np.exp(-c * (target * np.log(pred_p)).sum(axis=1))
    64.             sample_weight = sample_weight / np.sum(sample_weight) * self.n_samples
    65.  
    66.     @staticmethod
    67.     def softmax_func(x):
    68.         """
    69.         softmax函数，为避免上溢或下溢，对参数x做限制
    70.         :param x: 数组: batch_size * n_classes
    71.         :return:  1 * n_classes
    72.         """
    73.         exps = np.exp(x - np.max(x))  # 避免溢出，每个数减去其最大值
    74.         exp_sum = np.sum(exps, axis=1, keepdims=True)
    75.         return exps / exp_sum
    76.  
    77.     def predict_proba(self, x_test):
    78.         """
    79.         预测测试样本所属类别概率，软投票
    80.         :param x_test: 测试样本集
    81.         :return:
    82.         """
    83.         x_test = np.asarray(x_test)
    84.         C_x = np.zeros((x_test.shape[0], self.n_class))
    85.         for i in range(self.n_estimators):
    86.             y_prob = self.base_estimator[i].predict_proba(x_test)
    87.             np.clip(y_prob, np.finfo(y_prob.dtype).eps, None, out=y_prob)
    88.             y_ln = np.log(y_prob)
    89.             C_x += (self.n_class - 1) * (y_ln - np.sum(y_ln, axis=1, keepdims=True) / self.n_class)
    90.         return C_x
    91.  
    92.     def predict(self, x_test):
    93.         """
    94.         预测测试样本所属类别
    95.         :param x_test: 测试样本集
    96.         :return:
    97.         """
    98.         return np.argmax(self.predict_proba(x_test), axis=1)

    1.4 Adaboost分类算法测试

    test_adaboost_c.py

    1. from sklearn.datasets import make_classification
    2. from sklearn.metrics import classification_report
    3.  
    4. from ch4.decision_tree_C import DecisionTreeClassifier  # 基学习器，决策树
    5. from ch3.logistic_regression_2class import LogisticRegression  # 逻辑回归
    6. from ch6.svm_smo_classifier import SVMClassifier  # 支持向量机
    7. from adaboost_discrete_c import AdaBoostClassifier
    8. from ch8.plt_decision_function import plot_decision_function
    9.  
    10.  
    11. X, y = make_classification(n_samples=300, n_features=2, n_informative=1, n_redundant=0, n_repeated=0, n_classes=2,
    12.                            n_clusters_per_class=1, class_sep=1, random_state=42)
    13. # 同质：同种类型的基学习器
    14. base_tree = DecisionTreeClassifier(max_depth=3, is_feature_all_R=True, max_bins=20)
    15. ada_bc = AdaBoostClassifier(base_estimator=base_tree, n_estimators=10, learning_rate=1.0)
    16. ada_bc.fit(X, y)  # adaboost训练
    17. print("基学习器的权重系数：\n", ada_bc.estimator_weights)
    18. y_pred = ada_bc.predict(X)  # 预测类别
    19. print(classification_report(y, y_pred))
    20. plot_decision_function(X, y, ada_bc)
    21.  
    22. # 异质：不同类型的基学习器
    23. log_reg = LogisticRegression(batch_size=20, max_epochs=5)
    24. cart = DecisionTreeClassifier(max_depth=4, is_feature_all_R=True)
    25. svm = SVMClassifier(C=5.0, max_epochs=20)
    26. ada_bc2 = AdaBoostClassifier(base_estimator=[log_reg, cart, svm], learning_rate=1.0)
    27. ada_bc2.fit(X, y)  # adaboost训练
    28. print("异质基学习器的权重系数：", ada_bc2.estimator_weights)
    29. y_pred = ada_bc2.predict(X)  # 预测类别
    30. print(classification_report(y, y_pred))
    31. plot_decision_function(X, y, ada_bc2)
    32.  
    33.

     test_adaboost_c2.py

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. from sklearn.preprocessing import StandardScaler
    4. from sklearn.model_selection import KFold
    5. from sklearn.metrics import accuracy_score
    6. from sklearn.datasets import make_blobs
    7. from ch4.decision_tree_C import DecisionTreeClassifier
    8. from ch8.adaboost_discrete_c import AdaBoostClassifier
    9.  
    10.  
    11. X, y = make_blobs(n_samples=1000, n_features=10, centers=5, cluster_std=[1.5, 2, 0.9, 3, 2.8], random_state=0)
    12. X = StandardScaler().fit_transform(X)
    13.  
    14. base_em = DecisionTreeClassifier(max_depth=4, is_feature_all_R=True, max_bins=10)
    15. acc_scores = []  # 存储每次交叉验证的均分
    16. # 用10折交叉验证评估不同基学习器个数T下的分类正确率
    17. for n in range(1, 21):
    18.     scores = []  # 一次交叉验证的acc均值
    19.     k_fold = KFold(n_splits=10)
    20.     for idx_train, idx_test in k_fold.split(X, y):
    21.         classifier = AdaBoostClassifier(base_estimator=base_em, n_estimators=n, learning_rate=1)
    22.         classifier.fit(X[idx_train, :], y[idx_train])
    23.         y_test_pred = classifier.predict(X[idx_test, :])
    24.         scores.append(accuracy_score(y[idx_test], y_test_pred))
    25.     acc_scores.append(np.mean(scores))
    26.     print(n, ":", acc_scores[-1])
    27.  
    28. plt.figure(figsize=(7, 5))
    29. plt.plot(range(1, 21), acc_scores, "ko-", lw=1)
    30. plt.xlabel("Number of Estimations", fontdict={"fontsize": 12})
    31. plt.ylabel("Accuracy Score", fontdict={"fontsize": 12})
    32. plt.title("Cross Validation Scores of Different Number of Base Learners", fontdict={"fontsize": 14})
    33. plt.grid(ls=":")
    34. plt.show()
    35.

    1.5 Adaboost回归算法测试

    test_adaboost_regressor.py

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. from sklearn.datasets import fetch_california_housing
    4. from sklearn.metrics import r2_score
    5. from sklearn.preprocessing import StandardScaler
    6. from sklearn.model_selection import train_test_split
    7. from ch4.decision_tree_R import DecisionTreeRegression
    8. from ch8.adaboost_regressor import AdaBoostRegressior
    9.  
    10.  
    11. housing = fetch_california_housing()
    12. X, y = housing.data, housing.target
    13. # print(X.shape)
    14. # print(y.shape)
    15. X = StandardScaler().fit_transform(X)
    16. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
    17.  
    18. base_ht = DecisionTreeRegression(max_bins=50, max_depth=5)
    19. plt.figure(figsize=(14, 15))
    20.  
    21.  
    22. def train_plot(cs, loss, i):
    23.     abr = AdaBoostRegressior(base_estimator=base_ht, n_estimators=30,
    24.                              comb_strategy=cs, loss=loss)
    25.     abr.fit(X_train, y_train)
    26.     y_hat = abr.predict(X_test)
    27.     # print(r2_score(y_test, y_hat))
    28.     plt.subplots(231 + i)
    29.     idx = np.argsort(y_test)  # 对真值排序
    30.     plt.plot(y_test[idx], "k-", lw=1.5, label="Test True")
    31.     plt.plot(y_hat[idx], "r-", lw=1, label="Predict")
    32.     plt.legend(frameon=False)
    33.     plt.title("%s, %s, R2 = %.5f, MSE = %.5f" %
    34.               (cs, loss, r2_score(y_test, y_hat), ((y_test - y_hat) ** 2).mean()))
    35.     plt.xlabel("Test Samples Serial Number", fontdict={"fontsize": 12})
    36.     plt.ylabel("True VS Predict", fontdict={"fontsize": 12})
    37.     plt.grid(ls=":")
    38.     print(cs, loss)
    39.  
    40.  
    41. loss_func = ["linear", "square", "exp"]
    42. comb_strategy = ["weight_mean", "weight_median"]
    43. i = 0
    44. for loss in loss_func:
    45.     for cs in comb_strategy:
    46.         train_plot(cs, loss, i)
    47.         i += 1
    48. plt.show()

     1.6 SAMME算法测试

    test_samme_r_c.py

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. from sklearn.preprocessing import StandardScaler
    4. from sklearn.model_selection import KFold
    5. from sklearn.metrics import accuracy_score
    6. from sklearn.datasets import make_blobs
    7. # from ch4.decision_tree_C import DecisionTreeClassifierR
    8. from sklearn.tree import DecisionTreeClassifier
    9. from ch8.samme_r_muti_classifier import SAMMERClassifier
    10.  
    11.  
    12. X, y = make_blobs(n_samples=1000, n_features=10, centers=5, cluster_std=[1.5, 2, 0.9, 3, 2.8], random_state=0)
    13. X = StandardScaler().fit_transform(X)
    14.  
    15. base_em = DecisionTreeClassifier(max_depth=4)
    16. acc_scores = []  # 存储每次交叉验证的均分
    17. # 用10折交叉验证评估不同基学习器个数T下的分类正确率
    18. for n in range(1, 21):
    19.     scores = []  # 一次交叉验证的acc均值
    20.     k_fold = KFold(n_splits=10)
    21.     for idx_train, idx_test in k_fold.split(X, y):
    22.         classifier = SAMMERClassifier(base_estimator=base_em, n_estimators=n)
    23.         classifier.fit(X[idx_train, :], y[idx_train])
    24.         y_test_pred = classifier.predict(X[idx_test, :])
    25.         scores.append(accuracy_score(y[idx_test], y_test_pred))
    26.     acc_scores.append(np.mean(scores))
    27.     print(n, ":", acc_scores[-1])
    28.  
    29. plt.figure(figsize=(7, 5))
    30. plt.plot(range(1, 21), acc_scores, "ko-", lw=1)
    31. plt.xlabel("Number of Estimations", fontdict={"fontsize": 12})
    32. plt.ylabel("Accuracy Score", fontdict={"fontsize": 12})
    33. plt.title("Cross Validation Scores of Different Number of Base Learners", fontdict={"fontsize": 14})
    34. plt.grid(ls=":")
    35. plt.show()
    36.

    1.7 可视化分类边界函数

     plt_decision_function.py

    1. import matplotlib.pyplot as plt
    2. import numpy as np
    3.  
    4.  
    5. def plot_decision_function(X, y, clf, is_show=True):
    6.     """
    7.     可视化分类边界函数
    8.     :param X: 测试样本
    9.     :param y: 测试样本的类别
    10.     :param clf: 分类模型
    11.     :param is_show: 是否在当前显示图像，用于父函数绘制子图
    12.     :return:
    13.     """
    14.     if is_show:
    15.         plt.figure(figsize=(7, 5))
    16.     x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    17.     y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    18.     xi, yi = np.meshgrid(np.linspace(x_min, x_max, 100),
    19.                          np.linspace(y_min, y_max, 100))
    20.     y_pred = clf.predict(np.c_[xi.ravel(), yi.ravel()])  # 模型预测值
    21.     y_pred = y_pred.reshape(xi.shape)
    22.     plt.contourf(xi, yi, y_pred, cmap="winter", alpha=0.4)
    23.     plt.scatter(X[:, 0], X[:, 1], c=y, edgecolors="k")
    24.     plt.xlabel("Feature 1", fontdict={"fontsize": 12})
    25.     plt.ylabel("Feature 2", fontdict={"fontsize": 12})
    26.     plt.title("Model Classification Boundary", fontdict={"fontsize": 14})
    27.     if is_show:
    28.         plt.show()
    29.

    二、提升树算法boosting tree

    2.1 提升树回归算法

    boostingtree_r.py

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4.  
    5.  
    6. class BoostTreeRegressor:
    7.     """
    8.     提升树回归算法，采用平方误差损失
    9.     """
    10.     def __init__(self, base_estimator=None, n_estimators=10, learning_rate=1.0):
    11.         """
    12.         :param base_estimator: 基学习器
    13.         :param n_estimators: 基学习器的个数T
    14.         :param learning_rate: 学习率，降低后续训练的基分类器的权重，避免过拟合
    15.         """
    16.         self.base_estimator = base_estimator
    17.         self.n_estimators = n_estimators
    18.         self.learning_rate = learning_rate
    19.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    20.         if self.base_estimator is None:
    21.             self.base_estimator = DecisionTreeRegression(max_depth=2)
    22.         if type(base_estimator) != list:
    23.             # 同质（同种类型）的分类器，深拷贝
    24.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    25.                                    for _ in range(self.n_estimators)]
    26.         else:
    27.             # 异质（不同种类型）的分类器
    28.             self.n_estimators = len(self.base_estimator)
    29.  
    30.     def fit(self, x_train, y_train):
    31.         """
    32.         提升树的训练，针对每个基决策树算法，拟合上一轮的残差
    33.         :param x_train: 训练集
    34.         :param y_train: 目标集
    35.         :return:
    36.         """
    37.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    38.         # 1. 训练第一棵回归决策树，并预测
    39.         self.base_estimator[0].fit(x_train, y_train)
    40.         y_hat = self.base_estimator[0].predict(x_train)
    41.         y_residual = y_train - y_hat  # 残差，MSE的负梯度
    42.         # 2. 从第二棵树开始，每一轮拟合上一轮的残差
    43.         for idx in range(1, self.n_estimators):
    44.             self.base_estimator[idx].fit(x_train, y_residual)  # 拟合残差
    45.             # 累加第m-1棵树的预测值，当前模型是f_(m-1)
    46.             y_hat += self.base_estimator[idx].predict(x_train) * self.learning_rate
    47.             y_residual = y_train - y_hat  # 当前模型的残差
    48.  
    49.     def predict(self, x_test):
    50.         """
    51.         回归提升树的预测
    52.         :param x_test: 测试样本集
    53.         :return:
    54.         """
    55.         x_test = np.asarray(x_test)
    56.         y_hat_mat = np.sum([self.base_estimator[0].predict(x_test)] +
    57.                             [np.power(self.learning_rate, i) * self.base_estimator[i].predict(x_test)
    58.                              for i in range(1, self.n_estimators - 1)] +
    59.                             [self.base_estimator[-1].predict(x_test)], axis=0)
    60.         return y_hat_mat
    61.  
    62.

    2.2 梯度提升树分类算法

    gradientboosting_c

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4.  
    5.  
    6. class GradientBoostClassifier:
    7.     """
    8.     梯度提升树多分类算法：多分类也可用回归树来做，即训练与类别数相同的几组回归树，
    9.     每一组代表一个类别，然后对所有组的输出进行softmax操作将其转换为概率分布，
    10.     再通过交叉熵或者KL一类的损失函数求每棵树相应的负梯度，指导下一轮的训练。
    11.     """
    12.     def __init__(self, base_estimator=None, n_estimators=10, learning_rate=1.0):
    13.         """
    14.         :param base_estimator: 基学习器
    15.         :param n_estimators: 基学习器的个数T
    16.         :param learning_rate: 学习率，降低后续训练的基分类器的权重，避免过拟合
    17.         """
    18.         self.base_estimator = base_estimator
    19.         self.n_estimators = n_estimators
    20.         self.learning_rate = learning_rate
    21.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    22.         if self.base_estimator is None:
    23.             self.base_estimator = DecisionTreeRegression(max_depth=2)
    24.         if type(base_estimator) != list:
    25.             # 同质（同种类型）的分类器，深拷贝
    26.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    27.                                    for _ in range(self.n_estimators)]
    28.         else:
    29.             # 异质（不同种类型）的分类器
    30.             self.n_estimators = len(self.base_estimator)
    31.         self.base_estimators = []  # 扩展到class_num组分类器
    32.  
    33.     @staticmethod
    34.     def one_hot_encoding(target):
    35.         class_labels = np.unique(target)
    36.         target_y = np.zeros((len(target), len(class_labels)), dtype=np.int32)
    37.         for i, label in enumerate(target):
    38.             target_y[i, label] = 1  # 对应类别所在的列为1
    39.         return target_y
    40.  
    41.     @staticmethod
    42.     def softmax_func(x):
    43.         exps = np.exp(x - np.max(x))
    44.         return exps / np.sum(exps, axis=1, keepdims=True)
    45.  
    46.     def fit(self, x_train, y_train):
    47.         """
    48.         梯度提升分类算法的训练，共训练M * K个基学习器
    49.         :param x_train: 训练集
    50.         :param y_train: 目标集
    51.         :return:
    52.         """
    53.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    54.         class_num = len(np.unique(y_train))  # 类别数
    55.         y_encoded = self.one_hot_encoding(y_train)
    56.         # 深拷贝class_num组分类器，每组（每个类别）n_estimators个基学习器
    57.         # 假设是三分类：[[0, 1, 2, ..., 9], [10], [10]]
    58.         self.base_estimators = [copy.deepcopy(self.base_estimator) for _ in range(class_num)]
    59.  
    60.         # 初始化第一轮基学习器，针对每个类别，分别训练一个基学习器
    61.         y_hat_scores = []  # 用于存储每个类别的预测值
    62.         for c_idx in range(class_num):
    63.             self.base_estimators[c_idx][0].fit(x_train, y_encoded[:, c_idx])
    64.             y_hat_scores.append(self.base_estimators[c_idx][0].predict(x_train))
    65.         y_hat_scores = np.c_[y_hat_scores].T  # 把每个类别的预测值构成一列，(120, 3) (n_samples, class_num)
    66.         # print(np.asarray(y_hat_vals).shape)
    67.         grad_y = y_encoded - self.softmax_func(y_hat_scores)  # 按类别计算样本的负梯度值
    68.  
    69.         # 训练后续基学习器，共M - 1轮，每轮针对每个类别，分别训练一个基学习器
    70.         for idx in range(1, self.n_estimators):
    71.             y_hat_values = []  # 用于存储每个类别的预测值
    72.             for c_idx in range(class_num):
    73.                 self.base_estimators[c_idx][idx].fit(x_train, grad_y[:, c_idx])
    74.                 y_hat_values.append(self.base_estimators[c_idx][idx].predict(x_train))
    75.             y_hat_scores += np.c_[y_hat_values].T * self.learning_rate
    76.             # print(np.asarray(y_hat_vals).shape)
    77.             grad_y = y_encoded - self.softmax_func(y_hat_scores)  # 按类别计算样本的负梯度值
    78.  
    79.     def predict_proba(self, x_test):
    80.         """
    81.         预测测试样本所属类别的概率
    82.         :param x_test: 测试样本集
    83.         :return:
    84.         """
    85.         x_test = np.asarray(x_test)
    86.         y_hat_scores = []
    87.         for c_idx in range(len(self.base_estimators)):
    88.             # 取当前类别的M个基学习器
    89.             estimator = self.base_estimators[c_idx]
    90.             y_hat_scores.append(np.sum([estimator[0].predict(x_test)] +
    91.                                        [self.learning_rate * estimator[i].predict(x_test)
    92.                                        for i in range(1, self.n_estimators - 1)] +
    93.                                        [estimator[-1].predict(x_test)], axis=0))
    94.         # y_hat_scores的维度（3 * 30）
    95.         return self.softmax_func(np.c_[y_hat_scores].T)
    96.  
    97.     def predict(self, x_test):
    98.         """
    99.         预测测试样本所属类别，概率大的idx标记为类别
    100.         :param x_test: 测试样本集
    101.         :return:
    102.         """
    103.         print(self.predict_proba(x_test))
    104.         return np.argmax(self.predict_proba(x_test), axis=1)

     2.3 梯度提升树回归算法

    gradientboosting_r

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4.  
    5.  
    6. class GradientBoostRegressor:
    7.     """
    8.     梯度提升树回归算法，损失函数：五个，以损失函数在当前模型的负梯度近似为残差
    9.     """
    10.     def __init__(self, base_estimator=None, n_estimators=10, learning_rate=1.0,
    11.                  loss="ls", huber_threshold=0.1, quantile_threshold=0.5):
    12.         """
    13.         :param base_estimator: 基学习器
    14.         :param n_estimators: 基学习器的个数T
    15.         :param learning_rate: 学习率，降低后续训练的基分类器的权重，避免过拟合
    16.         """
    17.         self.base_estimator = base_estimator
    18.         self.n_estimators = n_estimators
    19.         self.learning_rate = learning_rate
    20.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    21.         if self.base_estimator is None:
    22.             self.base_estimator = DecisionTreeRegression(max_depth=2)
    23.         if type(base_estimator) != list:
    24.             # 同质（同种类型）的分类器，深拷贝
    25.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    26.                                    for _ in range(self.n_estimators)]
    27.         else:
    28.             # 异质（不同种类型）的分类器
    29.             self.n_estimators = len(self.base_estimator)
    30.         self.loss = loss  # 损失函数的类型
    31.         self.huber_threshold = huber_threshold  # 仅对Huber损失有效
    32.         self.quantile_threshold = quantile_threshold  # 仅对分位数损失函数有效
    33.  
    34.     def _cal_negative_gradient(self, y_true, y_pred):
    35.         """
    36.         计算负梯度值
    37.         :param y_true: 真值
    38.         :param y_pred: 预测值
    39.         :return:
    40.         """
    41.         if self.loss.lower() == "ls":  # MSE
    42.             return y_true - y_pred
    43.         elif self.loss.lower() == "lae":  # MAE
    44.             return np.sign(y_true - y_pred)
    45.         elif self.loss.lower() == "huber":  # 平滑平均绝对损失
    46.             return np.where(np.abs(y_true - y_pred) > self.huber_threshold,
    47.                             self.huber_threshold * np.sign(y_true - y_pred),
    48.                             y_true - y_pred)
    49.         elif self.loss.lower() == "quantile":  # 分位数损失
    50.             return np.where(y_true > y_pred, self.quantile_threshold,
    51.                             self.quantile_threshold - 1)
    52.         elif self.loss.lower() == "logcosh":  # 双曲余弦的对数的负梯度
    53.             return -np.tanh(y_pred - y_true)
    54.         else:
    55.             raise ValueError("仅限于ls、lae、huber、quantile和logcosh，选择有误...")
    56.  
    57.     def fit(self, x_train, y_train):
    58.         """
    59.         提升树的训练，针对每个基决策树算法，拟合上一轮的残差
    60.         1. 假设回归决策树以mse构建的，针对不同的损失函数，计算不同的基尼指数划分标准
    61.         2. 预测，集成，也根据不同的损失函数，预测叶子结点的输出...
    62.         :param x_train: 训练集
    63.         :param y_train: 目标集
    64.         :return:
    65.         """
    66.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    67.         # 1. 训练第一棵回归决策树，并预测
    68.         self.base_estimator[0].fit(x_train, y_train)
    69.         y_hat = self.base_estimator[0].predict(x_train)
    70.         y_residual = self._cal_negative_gradient(y_train, y_hat)  # 负梯度
    71.         # 2. 从第二棵树开始，每一轮拟合上一轮的残差
    72.         for idx in range(1, self.n_estimators):
    73.             self.base_estimator[idx].fit(x_train, y_residual)  # 拟合残差
    74.             # 累加第m-1棵树的预测值，当前模型是f_(m-1)
    75.             y_hat += self.base_estimator[idx].predict(x_train) * self.learning_rate
    76.             y_residual = self._cal_negative_gradient(y_train, y_hat)  # 负梯度
    77.  
    78.     def predict(self, x_test):
    79.         """
    80.         回归提升树的预测
    81.         :param x_test: 测试样本集
    82.         :return:
    83.         """
    84.         x_test = np.asarray(x_test)
    85.         y_hat_mat = np.sum([self.base_estimator[0].predict(x_test)] +
    86.                             [np.power(self.learning_rate, i) * self.base_estimator[i].predict(x_test)
    87.                              for i in range(1, self.n_estimators - 1)] +
    88.                             [self.base_estimator[-1].predict(x_test)], axis=0)
    89.         return y_hat_mat
    90.  
    91.

     2.4 提升树算法测试

    test_boosting_tree_r.py

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. from sklearn.datasets import fetch_california_housing
    4. from sklearn.metrics import r2_score
    5. from sklearn.preprocessing import StandardScaler
    6. from sklearn.model_selection import train_test_split
    7. from ch4.decision_tree_R import DecisionTreeRegression
    8. from ch8.boostingtree_r import BoostTreeRegressor
    9. from sklearn.tree import DecisionTreeRegressor
    10.  
    11.  
    12. # housing = fetch_california_housing()
    13. # X, y = housing.data[0:20000:100, :], housing.target[0:20000:100]
    14. # print(X.shape)
    15. # print(y.shape)
    16. # X = StandardScaler().fit_transform(X)
    17. # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
    18.  
    19. X = np.linspace(1, 10, 10).reshape(-1, 1)
    20. y = np.array([5.56, 5.70, 5.91, 6.40, 6.80, 7.05, 8.90, 8.70, 9.00, 9.05])
    21.  
    22. # base_ht = DecisionTreeRegression(max_bins=10, max_depth=1)
    23. base_ht = DecisionTreeRegressor(max_depth=1)
    24. # n_estimators = np.linspace(2, 31, 29, dtype=np.int32)
    25. # r2_scores = []
    26. # for n in n_estimators:
    27. #     btr = BoostTreeRegressior(base_estimator=base_ht, n_estimators=n)
    28. #     btr.fit(X_train, y_train)
    29. #     y_hat = btr.predict(X_test)
    30. #     # print(r2_score(y_test, y_hat))
    31. #     r2_scores.append(r2_score(y_test, y_hat))
    32. #     print(n, ":", r2_scores[-1])
    33.  
    34. r2_scores = []
    35. for n in range(1, 7):
    36.     btr = BoostTreeRegressor(base_estimator=base_ht, n_estimators=n)
    37.     btr.fit(X, y)
    38.     y_hat = btr.predict(X)
    39.     # print(r2_score(y_test, y_hat))
    40.     r2_scores.append(r2_score(y, y_hat))
    41.     print(n, ":", r2_scores[-1], np.sum((y - y_hat) ** 2))
    42.  
    43. # plt.figure(figsize=(7, 5))
    44. # plt.plot(n_estimators, r2_scores, "ko-", lw=1)
    45. # plt.show()
    46.  
    47. # idx = np.argsort(y_test)  # 对真值排序
    48. #
    49. # plt.figure(figsize=(7, 5))
    50. # plt.plot(y_test[idx], "k-", lw=1.5, label="Test True")
    51. # plt.plot(y_hat[idx], "r-", lw=1, label="Predict")
    52. # plt.legend(frameon=False)
    53. # plt.title("Regression Boosting Tree, R2 = %.5f, MSE = %.5f" %
    54. #           (r2_score(y_test, y_hat), ((y_test - y_hat) ** 2).mean()))
    55. # plt.xlabel("Test Samples Serial Number", fontdict={"fontsize": 12})
    56. # plt.ylabel("True VS Predict", fontdict={"fontsize": 12})
    57. # plt.grid(ls=":")
    58. #
    59. # plt.show()

     2.5 梯度提升树算法测试

    test_gradboost_c1.py

    1. from ch8.gradientboosting_c import GradientBoostClassifier
    2. from sklearn.datasets import load_iris, load_digits, load_breast_cancer
    3. from sklearn.decomposition import PCA
    4. from sklearn.preprocessing import StandardScaler
    5. from sklearn.model_selection import train_test_split
    6. from sklearn.metrics import classification_report
    7. from ch4.decision_tree_R import DecisionTreeRegression
    8. from sklearn.tree import DecisionTreeRegressor
    9.  
    10.  
    11. # iris = load_iris()
    12. # X, y = iris.data, iris.target
    13.  
    14. digits = load_digits()
    15. X, y = digits.data, digits.target
    16.  
    17. # bc_data = load_breast_cancer()
    18. # X, y = bc_data.data, bc_data.target
    19.  
    20. X = PCA(n_components=10).fit_transform(X)
    21. X = StandardScaler().fit_transform(X)
    22.  
    23. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42)
    24.  
    25. # base_es = DecisionTreeRegression(max_bins=50, max_depth=3)
    26. base_es = DecisionTreeRegressor(max_depth=3)
    27.  
    28. gbc = GradientBoostClassifier(base_estimator=base_es, n_estimators=50)
    29. gbc.fit(X_train, y_train)
    30. y_hat = gbc.predict(X_test)
    31. print(classification_report(y_test, y_hat))

     三、Bagging算法

    3.1 Bagging算法

    bagging_c_r.py

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4. from ch4.decision_tree_C import DecisionTreeClassifier
    5. from sklearn.metrics import accuracy_score, r2_score
    6.  
    7.  
    8. class BaggingClassifierRegressor:
    9.     """
    10.     1. Bagging的基本流程：采样出T个含m个训练样本的采样集，然后基于每个采样集训练出一个基学习器，再集成。
    11.     2. 预测输出进行结合：Bagging通常对分类任务采用简单投票法，对回归任务使用简单平均法。
    12.     3. 把回归任务与分类任务集成到一个算法中，右参数task来控制，包外估计OOB控制
    13.     """
    14.     def __init__(self, base_estimator=None, n_estimators=10, task="C", OOB=False):
    15.         """
    16.         :param base_estimator: 基学习器
    17.         :param n_estimators: 基学习器的个数T
    18.         :param task: 任务：C代表分类任务，R代表回归任务
    19.         :param OOB: 布尔变量，True表示进行包外估计
    20.         """
    21.         self.base_estimator = base_estimator
    22.         self.n_estimators = n_estimators
    23.         self.task = task
    24.         if task.lower() not in ["c", "r"]:
    25.             raise ValueError("Bagging任务仅限分类（C/c）、回归（R/r）")
    26.         # 如果不提供基学习器，则默认按照深度为2的决策树作为基分类器
    27.         if self.base_estimator is None:
    28.             if self.task.lower() == "c":
    29.                 self.base_estimator = DecisionTreeClassifier()
    30.             elif self.task.lower() == "r":
    31.                 self.base_estimator = DecisionTreeRegression()
    32.         if type(base_estimator) != list:
    33.             # 同质（同种类型）的分类器，深拷贝
    34.             self.base_estimator = [copy.deepcopy(self.base_estimator)
    35.                                    for _ in range(self.n_estimators)]
    36.         else:
    37.             # 异质（不同种类型）的分类器
    38.             self.n_estimators = len(self.base_estimator)
    39.         self.OOB = OOB  # 是否进行包外估计
    40.         self.oob_indices = []  # 保存每次有放回采样未被使用的样本索引
    41.         self.y_oob_hat = None  # 包括估计样本预测值（回归）或预测类别概率（分类）
    42.         self.oob_score = None  # 包外估计的评分，分类和回归
    43.  
    44.     def fit(self, x_train, y_train):
    45.         """
    46.         Bagging算法（包含分类和回归）的训练
    47.         :param x_train: 训练集
    48.         :param y_train: 目标集
    49.         :return:
    50.         """
    51.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    52.         n_samples = x_train.shape[0]
    53.         for estimator in self.base_estimator:
    54.             # 1. 有放回的随机重采样训练集
    55.             indices = np.random.choice(n_samples, n_samples, replace=True)  # 采样样本索引
    56.             indices = np.unique(indices)
    57.             x_bootstrap, y_bootstrap = x_train[indices, :], y_train[indices]
    58.             # 2. 基于采样数据，训练基学习器
    59.             estimator.fit(x_bootstrap, y_bootstrap)
    60.             # 存储每个基学习器未使用的样本索引
    61.             n_indices = set(np.arange(n_samples)).difference(set(indices))
    62.             self.oob_indices.append(list(n_indices))  # 每个基学习器未参与训练的样本索引
    63.  
    64.         # 3. 包外估计
    65.         if self.OOB:
    66.             if self.task.lower() == "c":
    67.                 self._oob_score_classifier(x_train, y_train)
    68.             else:
    69.                 self._oob_score_regressor(x_train, y_train)
    70.  
    71.     def _oob_score_classifier(self, x_train, y_train):
    72.         """
    73.         分类任务的包外估计
    74.         :param x_train:
    75.         :param y_train:
    76.         :return:
    77.         """
    78.         self.y_oob_hat, y_true = [], []
    79.         for i in range(x_train.shape[0]):  # 针对每个训练样本
    80.             y_hat_i = []  # 当前样本在每个基学习器下的预测概率，个数未必等于self.n_estimators
    81.             for idx in range(self.n_estimators):  # 针对每个基学习器
    82.                 if i in self.oob_indices[idx]:  # 如果该样本属于包外估计
    83.                     y_hat = self.base_estimator[idx].predict_proba(x_train[i, np.newaxis])
    84.                     y_hat_i.append(y_hat[0])
    85.             # print(y_hat_i)
    86.             if y_hat_i:  # 非空，计算各基学习器预测类别概率的均值
    87.                 self.y_oob_hat.append(np.mean(np.c_[y_hat_i], axis=0))
    88.                 y_true.append(y_train[i])  # 存储对应的真值
    89.         self.y_oob_hat = np.asarray(self.y_oob_hat)
    90.         self.oob_score = accuracy_score(y_true, np.argmax(self.y_oob_hat, axis=1))
    91.  
    92.     def _oob_score_regressor(self, x_train, y_train):
    93.         """
    94.         回归任务的包外估计
    95.         :param x_train:
    96.         :param y_train:
    97.         :return:
    98.         """
    99.         self.y_oob_hat, y_true = [], []
    100.         for i in range(x_train.shape[0]):  # 针对每个训练样本
    101.             y_hat_i = []  # 当前样本在每个基学习器下的预测概率，个数未必等于self.n_estimators
    102.             for idx in range(self.n_estimators):  # 针对每个基学习器
    103.                 if i in self.oob_indices[idx]:  # 如果该样本属于包外估计
    104.                     y_hat = self.base_estimator[idx].predict(x_train[i, np.newaxis])
    105.                     y_hat_i.append(y_hat[0])
    106.             # print(y_hat_i)
    107.             if y_hat_i:  # 非空，计算各基学习器预测类别概率的均值
    108.                 self.y_oob_hat.append(np.mean(y_hat_i))
    109.                 y_true.append(y_train[i])  # 存储对应的真值
    110.         self.y_oob_hat = np.asarray(self.y_oob_hat)
    111.         self.oob_score = r2_score(y_true, self.y_oob_hat)
    112.  
    113.     def predict_proba(self, x_test):
    114.         """
    115.         分类任务中测试样本所属类别的概率预测
    116.         :param x_test:
    117.         :return:
    118.         """
    119.         if self.task.lower() != "c":
    120.             raise ValueError("predict_proba()仅适用于分类任务。")
    121.         x_test = np.asarray(x_test)
    122.         y_test_hat = []  # 用于存储测试样本所属类别概率
    123.         for estimator in self.base_estimator:
    124.             y_test_hat.append(estimator.predict_proba(x_test))
    125.         # print(y_test_hat)
    126.         return np.mean(y_test_hat, axis=0)
    127.  
    128.     def predict(self, x_test):
    129.         """
    130.         分类任务：预测测试样本所属类别，类别概率大者索引为所属类别
    131.         回归任务：预测测试样本，对每个基学习器预测值简单平均
    132.         :param x_test:
    133.         :return:
    134.         """
    135.         if self.task.lower() == "c":
    136.             return np.argmax(self.predict_proba(x_test), axis=1)
    137.         elif self.task.lower() == "r":
    138.             y_hat = []  # 预测值
    139.             for estimator in self.base_estimator:
    140.                 y_hat.append(estimator.predict(x_test))
    141.             return np.mean(y_hat, axis=0)

    3.2 Bagging算法测试

    test_bagging_c1.py

    1. from sklearn.datasets import load_iris
    2. from ch8.bagging_c_r import BaggingClassifierRegressor
    3. from ch4.decision_tree_C import DecisionTreeClassifier
    4. from sklearn.metrics import classification_report
    5. from sklearn.model_selection import train_test_split
    6. from sklearn.preprocessing import StandardScaler
    7.  
    8.  
    9. iris = load_iris()
    10. X, y = iris.data, iris.target
    11. X = StandardScaler().fit_transform(X)
    12. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, shuffle=True, random_state=42)
    13.  
    14. base_es = DecisionTreeClassifier(max_depth=10, max_bins=50, is_feature_all_R=True)
    15. bagcr = BaggingClassifierRegressor(base_estimator=base_es, n_estimators=20, task="c", OOB=True)
    16. bagcr.fit(X_train, y_train)
    17. y_hat = bagcr.predict(X_test)
    18. print(classification_report(y_test, y_hat))
    19. print("包外估计的精度：", bagcr.oob_score)

     test_bagging_c2.py

    1. from sklearn.datasets import load_iris
    2. from ch8.bagging_c_r import BaggingClassifierRegressor
    3. from ch4.decision_tree_C import DecisionTreeClassifier
    4. from sklearn.metrics import classification_report, accuracy_score
    5. from sklearn.model_selection import train_test_split
    6. from sklearn.preprocessing import StandardScaler, LabelEncoder
    7. import pandas as pd
    8. import numpy as np
    9. import matplotlib.pyplot as plt
    10.  
    11.  
    12. nursery = pd.read_csv("../ch4/data/nursery.csv").dropna()
    13. X, y = np.asarray(nursery.iloc[:, :-1]), np.asarray(nursery.iloc[:, -1])
    14. y = LabelEncoder().fit_transform(y)
    15.  
    16. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, shuffle=True, random_state=42)
    17.  
    18. base_es = DecisionTreeClassifier(max_depth=10)
    19. bagcr = BaggingClassifierRegressor(base_estimator=base_es, n_estimators=30, task="c")
    20. bagcr.fit(X_train, y_train)
    21. y_hat = bagcr.predict(X_test)
    22. print(classification_report(y_test, y_hat))
    23. # print("包外估计的精度：", bagcr.oob_score)
    24.  
    25. y_test_scores = []
    26. for i in range(30):
    27.     bagcr = BaggingClassifierRegressor(base_estimator=base_es, n_estimators=1, task="c")
    28.     bagcr.fit(X_train, y_train)
    29.     y_hat = bagcr.predict(X_test)
    30.     y_test_scores.append(accuracy_score(y_test, y_hat))
    31.  
    32. plt.figure(figsize=(7, 5))
    33. plt.plot(range(1, 31), y_test_scores, "ko-", lw=1.5)
    34. plt.xlabel("Training Times", fontsize=12)
    35. plt.ylabel("Test Accuracy", fontsize=12)
    36. plt.grid(ls=":")
    37. plt.show()
    38.  
    39.

     test_bagging_r.py

    1. import numpy as np
    2. import matplotlib.pyplot as plt
    3. from ch4.decision_tree_R import DecisionTreeRegression
    4. from ch8.bagging_c_r import BaggingClassifierRegressor
    5. from sklearn.metrics import r2_score
    6.  
    7.  
    8. f = lambda x: 0.5 * np.exp(-(x + 3) ** 2) + np.exp(-x ** 2) + 1.5 * np.exp(-(x - 3) ** 2)
    9.  
    10. np.random.seed(0)
    11. N = 200
    12. X = np.random.rand(N) * 10 - 5
    13. X = np.sort(X)
    14. y = f(X) + 0.05 * np.random.randn(N)
    15. X = X.reshape(-1, 1)
    16. # print(X)
    17.  
    18. base_estimator = DecisionTreeRegression(max_bins=30, max_depth=8)
    19. model = BaggingClassifierRegressor(base_estimator=base_estimator, n_estimators=100, task="r")
    20. model.fit(X, y)
    21.  
    22. X_test = np.linspace(1.1 * X.min(axis=0), 1.1 * X.max(axis=0), 1000).reshape(-1, 1)
    23.  
    24. y_bagging_hat = model.predict(X_test)
    25.  
    26. base_estimator.fit(X, y)
    27. y_cart_hat = base_estimator.predict(X_test)
    28.  
    29. plt.figure(figsize=(7, 5))
    30. plt.scatter(X, y, s=10, c="k", label="Raw Data")
    31. plt.plot(X_test, f(X_test), "k-", lw=1.5, label="True F(x)")
    32. plt.plot(X_test, y_bagging_hat, "r-", label="Bagging(R2 = %.5f)" % r2_score(f(X_test), y_bagging_hat))
    33. plt.plot(X_test, y_cart_hat, "b-", label="CART(R2 = %.5f)" % r2_score(f(X_test), y_cart_hat))
    34. plt.legend(frameon=False)
    35. plt.xlabel("X", fontsize=12)
    36. plt.ylabel("Y", fontsize=12)
    37. plt.grid(ls=":")
    38. plt.title("Bagging(100 estimators) VS CART Regression", fontsize=14)
    39. plt.show()

    四、随机森林算法

    4.1 随机森林算法

     rf_classifier_regressor.py

    1. import numpy as np
    2. import copy
    3. from ch4.decision_tree_R import DecisionTreeRegression  # CART
    4. from ch4.decision_tree_C import DecisionTreeClassifier
    5. from sklearn.metrics import accuracy_score, r2_score
    6.  
    7.  
    8. class RandomForestClassifierRegressor:
    9.     """
    10.     随机森林RF是Bagging的一个扩展变体。 RF在以决策树为基学习器构建Bagging集成的基础上，
    11.     进一步在决策树的训练过程中引入了随机属性选择， 即对训练样本和输入变量增加随机扰动。
    12.     """
    13.     def __init__(self, base_estimator=None, n_estimators=10, feature_sampling_rate=0.5,
    14.                  task="C", OOB=False, feature_importance=False):
    15.         """
    16.         :param base_estimator: 基学习器
    17.         :param n_estimators: 基学习器的个数T
    18.         :param task: 任务：C代表分类任务，R代表回归任务
    19.         :param OOB: 布尔变量，True表示进行包外估计
    20.         :param feature_sampling_rate: 特征变量的抽样率
    21.         :param feature_importance: 布尔变量，表示是否进行特征重要性的评估
    22.         """
    23.         self.base_estimator = base_estimator
    24.         self.n_estimators = n_estimators
    25.         self.feature_sampling_rate = feature_sampling_rate
    26.         if task.lower() not in ["c", "r"]:
    27.             raise ValueError("Bagging任务仅限分类（C/c）、回归（R/r）")
    28.         self.task = task
    29.         # 如果不提供基学习器，则默认决策树作为基分类器
    30.         if self.base_estimator is None:
    31.             if self.task.lower() == "c":
    32.                 base_estimator = DecisionTreeClassifier()
    33.             elif self.task.lower() == "r":
    34.                 base_estimator = DecisionTreeRegression()
    35.         self.base_estimator = [copy.deepcopy(base_estimator)
    36.                                for _ in range(self.n_estimators)]
    37.         self.OOB = OOB  # 是否进行包外估计
    38.         self.oob_indices = []  # 保存每次有放回采样未被使用的样本索引
    39.         self.y_oob_hat = None  # 包括估计样本预测值（回归）或预测类别概率（分类）
    40.         self.oob_score = None  # 包外估计的评分，分类和回归
    41.         self.feature_importance = feature_importance
    42.         self.feature_importance_scores = None  # 特征变量的重要性评分
    43.         self.feature_importance_indices = []  # 针对每个基学习器，存储特征变量的抽样索引
    44.  
    45.     def fit(self, x_train, y_train):
    46.         """
    47.         随机森林算法（包含分类和回归）的训练
    48.         :param x_train: 训练集
    49.         :param y_train: 目标集
    50.         :return:
    51.         """
    52.         x_train, y_train = np.asarray(x_train), np.asarray(y_train)
    53.         n_samples, n_features = x_train.shape
    54.         for estimator in self.base_estimator:
    55.             # 1. 有放回的随机重采样训练集
    56.             indices = np.random.choice(n_samples, n_samples, replace=True)  # 采样样本索引
    57.             indices = np.unique(indices)
    58.             x_bootstrap, y_bootstrap = x_train[indices, :], y_train[indices]
    59.             # 2. 对特征属性变量进行抽样
    60.             fb_num = int(self.feature_sampling_rate * n_features)  # 抽样特征数
    61.             feature_idx = np.random.choice(n_features, fb_num, replace=False)  # 不放回
    62.             self.feature_importance_indices.append(feature_idx)
    63.             x_bootstrap = x_bootstrap[:, feature_idx]  # 获取特征变量抽样后的训练样本
    64.             # 3. 基于采样数据，训练基学习器
    65.             estimator.fit(x_bootstrap, y_bootstrap)
    66.             # 存储每个基学习器未使用的样本索引
    67.             n_indices = set(np.arange(n_samples)).difference(set(indices))
    68.             self.oob_indices.append(list(n_indices))  # 每个基学习器未参与训练的样本索引
    69.  
    70.         # 4. 包外估计
    71.         if self.OOB:
    72.             if self.task.lower() == "c":
    73.                 self._oob_score_classifier(x_train, y_train)
    74.             else:
    75.                 self._oob_score_regressor(x_train, y_train)
    76.  
    77.         # 5. 特征重要性估计
    78.         if self.feature_importance:
    79.             if self.task.lower() == "c":
    80.                 self._feature_importance_score_classifier(x_train, y_train)
    81.             else:
    82.                 self._feature_importance_score_regressor(x_train, y_train)
    83.  
    84.     def _oob_score_classifier(self, x_train, y_train):
    85.         """
    86.         分类任务的包外估计
    87.         :param x_train:
    88.         :param y_train:
    89.         :return:
    90.         """
    91.         self.y_oob_hat, y_true = [], []
    92.         for i in range(x_train.shape[0]):  # 针对每个训练样本
    93.             y_hat_i = []  # 当前样本在每个基学习器下的预测概率，个数未必等于self.n_estimators
    94.             for idx in range(self.n_estimators):  # 针对每个基学习器
    95.                 if i in self.oob_indices[idx]:  # 如果该样本属于包外估计
    96.                     x_sample = x_train[i, self.feature_importance_indices[idx]]
    97.                     y_hat = self.base_estimator[idx].predict_proba(x_sample.reshape(1, -1))
    98.                     y_hat_i.append(y_hat[0])
    99.             # print(y_hat_i)
    100.             if y_hat_i:  # 非空，计算各基学习器预测类别概率的均值
    101.                 self.y_oob_hat.append(np.mean(np.c_[y_hat_i], axis=0))
    102.                 y_true.append(y_train[i])  # 存储对应的真值
    103.         self.y_oob_hat = np.asarray(self.y_oob_hat)
    104.         self.oob_score = accuracy_score(y_true, np.argmax(self.y_oob_hat, axis=1))
    105.  
    106.     def _oob_score_regressor(self, x_train, y_train):
    107.         """
    108.         回归任务的包外估计
    109.         :param x_train:
    110.         :param y_train:
    111.         :return:
    112.         """
    113.         self.y_oob_hat, y_true = [], []
    114.         for i in range(x_train.shape[0]):  # 针对每个训练样本
    115.             y_hat_i = []  # 当前样本在每个基学习器下的预测概率，个数未必等于self.n_estimators
    116.             for idx in range(self.n_estimators):  # 针对每个基学习器
    117.                 if i in self.oob_indices[idx]:  # 如果该样本属于包外估计
    118.                     x_sample = x_train[i, self.feature_importance_indices[idx]]
    119.                     y_hat = self.base_estimator[idx].predict(x_sample.reshape(1, -1))
    120.                     y_hat_i.append(y_hat[0])
    121.             # print(y_hat_i)
    122.             if y_hat_i:  # 非空，计算各基学习器预测类别概率的均值
    123.                 self.y_oob_hat.append(np.mean(y_hat_i))
    124.                 y_true.append(y_train[i])  # 存储对应的真值
    125.         self.y_oob_hat = np.asarray(self.y_oob_hat)
    126.         self.oob_score = r2_score(y_true, self.y_oob_hat)
    127.  
    128.     def _feature_importance_score_classifier(self, x_train, y_train):
    129.         """
    130.         分类问题的特征变量重要性评估计算
    131.         :param x_train:
    132.         :param y_train:
    133.         :return:
    134.         """
    135.         n_feature = x_train.shape[1]
    136.         self.feature_importance_scores = np.zeros(n_feature)  # 特征变量重要性评分
    137.         for f_j in range(n_feature):  # 针对每个特征变量
    138.             f_j_scores = []  # 当前第j个特征变量在所有基学习器预测的OOB误差变化
    139.             for idx, estimator in enumerate(self.base_estimator):
    140.                 f_s_indices = list(self.feature_importance_indices[idx])  # 获取当前基学习器的特征变量索引
    141.                 if f_j in f_s_indices:  # 表示当前基学习器中存在第j个特征变量
    142.                     # 1. 计算基于OOB的测试误差error
    143.                     x_samples = x_train[self.oob_indices[idx], :][:, f_s_indices]  # OOB样本以及特征抽样
    144.                     y_hat = estimator.predict(x_samples)
    145.                     error = 1 - accuracy_score(y_train[self.oob_indices[idx]], y_hat)
    146.                     # 2. 计算第j个特征随机打乱顺序后的测试误差
    147.                     np.random.shuffle(x_samples[:, f_s_indices.index(f_j)])  # 原地打乱第j个特征变量取值，其他特征取值不变
    148.                     y_hat_j = estimator.predict(x_samples)
    149.                     error_j = 1 - accuracy_score(y_train[self.oob_indices[idx]], y_hat_j)
    150.                     f_j_scores.append(error_j - error)
    151.             # 3. 计算所有基学习器对当前第j个特征评分的均值
    152.             self.feature_importance_scores[f_j] = np.mean(f_j_scores)
    153.         return self.feature_importance_scores
    154.  
    155.     def _feature_importance_score_regressor(self, x_train, y_train):
    156.         """
    157.         回归任务的特征变量重要性评估计算
    158.         :param x_train:
    159.         :param y_train:
    160.         :return:
    161.         """
    162.         n_feature = x_train.shape[1]
    163.         self.feature_importance_scores = np.zeros(n_feature)  # 特征变量重要性评分
    164.         for f_j in range(n_feature):  # 针对每个特征变量
    165.             f_j_scores = []  # 当前第j个特征变量在所有基学习器预测的OOB误差变化
    166.             for idx, estimator in enumerate(self.base_estimator):
    167.                 f_s_indices = list(self.feature_importance_indices[idx])  # 获取当前基学习器的特征变量索引
    168.                 if f_j in f_s_indices:  # 表示当前基学习器中存在第j个特征变量
    169.                     # 1. 计算基于OOB的测试误差error
    170.                     x_samples = x_train[self.oob_indices[idx], :][:, f_s_indices]  # OOB样本以及特征抽样
    171.                     y_hat = estimator.predict[x_samples]
    172.                     error = 1 - r2_score(y_train[self.oob_indices[idx]], y_hat)
    173.                     # 2. 计算第j个特征随机打乱顺序后的测试误差
    174.                     np.random.shuffle(x_samples[:, f_s_indices.index(f_j)])  # 原地打乱第j个特征变量取值，其他特征取值不变
    175.                     y_hat_j = estimator.predict[x_samples]
    176.                     error_j = 1 - r2_score(y_train[self.oob_indices[idx]], y_hat_j)
    177.                     f_j_scores.append(error_j - error)
    178.             # 3. 计算所有基学习器对当前第j个特征评分的均值
    179.             self.feature_importance_scores[f_j] = np.mean(f_j_scores)
    180.         return self.feature_importance_scores
    181.  
    182.     def predict_proba(self, x_test):
    183.         """
    184.         分类任务中测试样本所属类别的概率预测
    185.         :param x_test:
    186.         :return:
    187.         """
    188.         if self.task.lower() != "c":
    189.             raise ValueError("predict_proba()仅适用于分类任务。")
    190.         x_test = np.asarray(x_test)
    191.         y_test_hat = []  # 用于存储测试样本所属类别概率
    192.         for idx, estimator in enumerate(self.base_estimator):
    193.             x_test_bootstrap = x_test[:, self.feature_importance_indices[idx]]
    194.             y_test_hat.append(estimator.predict_proba(x_test_bootstrap))
    195.         # print(y_test_hat)
    196.         return np.mean(y_test_hat, axis=0)
    197.  
    198.     def predict(self, x_test):
    199.         """
    200.         分类任务：预测测试样本所属类别，类别概率大者索引为所属类别
    201.         回归任务：预测测试样本，对每个基学习器预测值简单平均
    202.         :param x_test:
    203.         :return:
    204.         """
    205.         if self.task.lower() == "c":
    206.             return np.argmax(self.predict_proba(x_test), axis=1)
    207.         elif self.task.lower() == "r":
    208.             y_hat = []  # 预测值
    209.             for idx, estimator in enumerate(self.base_estimator):
    210.                 x_test_bootstrap = x_test[:, self.feature_importance_indices[idx]]
    211.                 y_hat.append(estimator.predict(x_test_bootstrap))
    212.             return np.mean(y_hat, axis=0)

     4.2 随机森林算法测试

    test_rf_c1.py

     

    1. from sklearn.datasets import load_iris, load_wine, load_digits
    2. from ch8.randomforest.rf_classifier_regressor import RandomForestClassifierRegressor
    3. # from ch4.decision_tree_C import DecisionTreeClassifier
    4. from sklearn.tree import DecisionTreeClassifier
    5. from sklearn.metrics import classification_report
    6. from sklearn.model_selection import train_test_split
    7. from sklearn.preprocessing import StandardScaler
    8. import matplotlib.pyplot as plt
    9. import seaborn as sns
    10. import pandas as pd
    11.  
    12.  
    13. iris = load_iris()
    14. X, y = iris.data, iris.target
    15. X = StandardScaler().fit_transform(X)
    16. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, shuffle=True, random_state=42)
    17.  
    18. # base_es = DecisionTreeClassifier(max_depth=10, max_bins=50, is_feature_all_R=True)
    19. base_es = DecisionTreeClassifier(max_depth=10)
    20. rf_model = RandomForestClassifierRegressor(base_estimator=base_es, n_estimators=30,
    21.                                         task="c", OOB=True, feature_importance=True)
    22. rf_model.fit(X_train, y_train)
    23. y_hat = rf_model.predict(X_test)
    24. print(classification_report(y_test, y_hat))
    25. print("包外估计的精度：", rf_model.oob_score)
    26. print("特征重要性评分：", rf_model.feature_importance_scores)
    27.  
    28. plt.figure(figsize=(9, 5))
    29. data_pd = pd.DataFrame([iris.feature_names, rf_model.feature_importance_scores]).T
    30. data_pd.columns = ["Feature Names", "Importance"]
    31. sns.barplot(x="Importance", y="Feature Names", data=data_pd)
    32. plt.title("Iris DataSet Feature Importance Scores", fontdict={"fontsize": 14})
    33. plt.grid(ls=":")
    34. print(data_pd)
    35. plt.show()
    36.

     

  • 相关阅读:
    把key-value键值对的数据转换成xml格式，并根据key来匹配元素
    暴雨天，看天翼云如何“快准稳”防涝
    MySQL(7) Innodb 原理和日志
    Java 第三阶段增强分析需求，代码实现能力【JDBC】
    FreeRTOS教程7 事件组
    golang pprof
    编写方法将一个数组扁平化并且去重和递增排序
    软考网络工程师笔记-分值分布
    （主）9.26锁存器&状态机方法、题解大综合（加码加码加码）
    镜头下的光学
  • 原文地址：https://blog.csdn.net/2302_78896863/article/details/136423203