机器学习第6章决策树

参考：作者的jupyter notebook
chapter 6 – decision trees

1. 保存图片

   from __future__ import division, print_function, unicode_literals
   import numpy as np
   import matplotlib as mpl
   import matplotlib.pyplot as plt
   import os
   np.random.seed(42)
   
   mpl.rc('axes', labelsize=14)
   mpl.rc('xtick', labelsize=12)
   mpl.rc('ytick', labelsize=12)
   
   # where to save the figures
   project_root_dir = "images"
   chapter_id = "decision_trees"
   
   def save_fig(fig_id, tight_layout=true):
       path = os.path.join(project_root_dir, chapter_id, fig_id + ".png")
       print("saving figure", fig_id)
       if tight_layout:
           plt.tight_layout()
       plt.savefig(path, format='png', dpi=600)
   

决策树训练和可视化

2. 要了解决策树，让我们先构建一个决策树，看看它是如何做出预测的。下面的代码在鸢尾花数据集（见第4章）上训练了一个decisiontreeclassifier：

   from sklearn.datasets import load_iris
   from sklearn.tree import decisiontreeclassifier
   
   iris = load_iris()
   x = iris.data[:, 2:] # petal length and width
   y = iris.target
   
   tree_clf = decisiontreeclassifier(max_depth=2, random_state=42)
   tree_clf.fit(x, y)
   #print(tree_clf.fit(x, y))
   

3. 要将决策树可视化，首先，使用export_graphviz（）方法输出一个图形定义文件，命名为iris_tree.dot：

   from sklearn.tree import export_graphviz
   
   export_graphviz(
           tree_clf,
           out_file=image_path("iris_tree.dot"),
           feature_names=iris.feature_names[2:],
           class_names=iris.target_names,
           rounded=true,
           filled=true
       )
   #下面这行命令将.dot文件转换为.png图像文件：
   #$ dot -tpng iris_tree.dot -o iris_tree.png
   

做出预测

```
from matplotlib.colors import listedcolormap

def plot_decision_boundary(clf, x, y, axes=[0, 7.5, 0, 3], iris=true, legend=false, plot_training=true):
    x1s = np.linspace(axes[0], axes[1], 100)
    x2s = np.linspace(axes[2], axes[3], 100)
    x1, x2 = np.meshgrid(x1s, x2s)
    x_new = np.c_[x1.ravel(), x2.ravel()]
    y_pred = clf.predict(x_new).reshape(x1.shape)
    custom_cmap = listedcolormap(['#fafab0','#9898ff','#a0faa0'])
    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
    if not iris:
        custom_cmap2 = listedcolormap(['#7d7d58','#4c4c7f','#507d50'])
        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
    if plot_training:
        plt.plot(x[:, 0][y==0], x[:, 1][y==0], "yo", label="iris-setosa")
        plt.plot(x[:, 0][y==1], x[:, 1][y==1], "bs", label="iris-versicolor")
        plt.plot(x[:, 0][y==2], x[:, 1][y==2], "g^", label="iris-virginica")
        plt.axis(axes)
    if iris:
        plt.xlabel("petal length", fontsize=14)
        plt.ylabel("petal width", fontsize=14)
    else:
        plt.xlabel(r"$x_1$", fontsize=18)
        plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
    if legend:
        plt.legend(loc="lower right", fontsize=14)

plt.figure(figsize=(8, 4))
plot_decision_boundary(tree_clf, x, y)
plt.plot([2.45, 2.45], [0, 3], "k-", linewidth=2)
plt.plot([2.45, 7.5], [1.75, 1.75], "k--", linewidth=2)
plt.plot([4.95, 4.95], [0, 1.75], "k:", linewidth=2)
plt.plot([4.85, 4.85], [1.75, 3], "k:", linewidth=2)
plt.text(1.40, 1.0, "depth=0", fontsize=15)
plt.text(3.2, 1.80, "depth=1", fontsize=13)
plt.text(4.05, 0.5, "(depth=2)", fontsize=11)

save_fig("decision_tree_decision_boundaries_plot")
plt.show()
```

估算类别概率

4. 决策树同样可以估算某个实例属于特定类别k的概率

   #print(tree_clf.predict_proba([[5, 1.5]]))
   #print(tree_clf.predict([[5, 1.5]]))0
   

cart训练算法

scikit-learn使用的是分类与回归树（classification and regression tree，简称cart）算法来训练决策树（也叫作“生长”树）。

计算复杂度

基尼不纯度还是信息熵

正则化超参数

```
from sklearn.datasets import make_moons
xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)

deep_tree_clf1 = decisiontreeclassifier(random_state=42)
deep_tree_clf2 = decisiontreeclassifier(min_samples_leaf=4, random_state=42)
deep_tree_clf1.fit(xm, ym)
deep_tree_clf2.fit(xm, ym)

plt.figure(figsize=(11, 4))
plt.subplot(121)
plot_decision_boundary(deep_tree_clf1, xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=false)
plt.title("no restrictions", fontsize=16)
plt.subplot(122)
plot_decision_boundary(deep_tree_clf2, xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=false)
plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)

save_fig("min_samples_leaf_plot正则化超参数")
plt.show()
```

左图使用默认参数（即无约束）来训练决策树，右图的决策树应用min_samples_leaf=4进行训练。很明显，左图模型过度拟合，右图的泛化效果更佳。

回归

5. 决策树也可以执行回归任务。我们用scikit_learn的decisiontreeregressor来构建一个回归树，在一个带噪声的二次数据集上进行训练，其中max_depth=2：

   np.random.seed(42)
   m = 200
   x = np.random.rand(m, 1)
   y = 4 * (x - 0.5) ** 2
   y = y + np.random.randn(m, 1) / 10
   
   from sklearn.tree import decisiontreeregressor
   
   tree_reg = decisiontreeregressor(max_depth=2, random_state=42)
   tree_reg.fit(x, y)
   #print(tree_reg.fit(x, y))
   

6. 两个决策树回归模型的预测对比

   from sklearn.tree import decisiontreeregressor
   
   tree_reg1 = decisiontreeregressor(random_state=42, max_depth=2)
   tree_reg2 = decisiontreeregressor(random_state=42, max_depth=3)
   tree_reg1.fit(x, y)
   tree_reg2.fit(x, y)
   
   def plot_regression_predictions(tree_reg, x, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
       x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
       y_pred = tree_reg.predict(x1)
       plt.axis(axes)
       plt.xlabel("$x_1$", fontsize=18)
       if ylabel:
           plt.ylabel(ylabel, fontsize=18, rotation=0)
       plt.plot(x, y, "b.")
       plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
   
   plt.figure(figsize=(11, 4))
   plt.subplot(121)
   plot_regression_predictions(tree_reg1, x, y)
   for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
       plt.plot([split, split], [-0.2, 1], style, linewidth=2)
   plt.text(0.21, 0.65, "depth=0", fontsize=15)
   plt.text(0.01, 0.2, "depth=1", fontsize=13)
   plt.text(0.65, 0.8, "depth=1", fontsize=13)
   plt.legend(loc="upper center", fontsize=18)
   plt.title("max_depth=2", fontsize=14)
   
   plt.subplot(122)
   plot_regression_predictions(tree_reg2, x, y, ylabel=none)
   for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
       plt.plot([split, split], [-0.2, 1], style, linewidth=2)
   for split in (0.0458, 0.1298, 0.2873, 0.9040):
       plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
   plt.text(0.3, 0.5, "depth=2", fontsize=13)
   plt.title("max_depth=3", fontsize=14)
   
   save_fig("tree_regression_plot两个决策树回归模型的预测对比")
   plt.show()
   

不稳定性

7. 对数据旋转敏感

   np.random.seed(6)
   xs = np.random.rand(100, 2) - 0.5
   ys = (xs[:, 0] > 0).astype(np.float32) * 2
   
   angle = np.pi / 4
   rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
   xsr = xs.dot(rotation_matrix)
   
   tree_clf_s = decisiontreeclassifier(random_state=42)
   tree_clf_s.fit(xs, ys)
   tree_clf_sr = decisiontreeclassifier(random_state=42)
   tree_clf_sr.fit(xsr, ys)
   
   plt.figure(figsize=(11, 4))
   plt.subplot(121)
   plot_decision_boundary(tree_clf_s, xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=false)
   plt.subplot(122)
   plot_decision_boundary(tree_clf_sr, xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=false)
   
   save_fig("sensitivity_to_rotation_plot对数据旋转敏感")
   plt.show()
   

8. 对训练集细节敏感

   x[(x[:, 1]==x[:, 1][y==1].max()) & (y==1)] # widest iris-versicolor flower
   
   not_widest_versicolor = (x[:, 1]!=1.8) | (y==2)
   x_tweaked = x[not_widest_versicolor]
   y_tweaked = y[not_widest_versicolor]
   
   tree_clf_tweaked = decisiontreeclassifier(max_depth=2, random_state=40)
   tree_clf_tweaked.fit(x_tweaked, y_tweaked)
   
   plt.figure(figsize=(8, 4))
   plot_decision_boundary(tree_clf_tweaked, x_tweaked, y_tweaked, legend=false)
   plt.plot([0, 7.5], [0.8, 0.8], "k-", linewidth=2)
   plt.plot([0, 7.5], [1.75, 1.75], "k--", linewidth=2)
   plt.text(1.0, 0.9, "depth=0", fontsize=15)
   plt.text(1.0, 1.80, "depth=1", fontsize=13)
   
   save_fig("decision_tree_inst    ability_plot对训练集细节敏感")
   plt.show()