Lab 3: Classification (cont.)

GradientBoosting

Gradient Boosting, implemented through the GradientBoostingClassifier in the sklearn.ensemble package, is anoother powerful classification technique.

Like random forests, it trains a set of n decision trees, that are combined in an ensemble of n_estimators. Each tree, however, is trained over the original dataset and usually has a fixed maximum depth. Beside many other parameters we can choose the maximum tree depth, through the max_depth parameter and the learning rate to use, through the learning_rate parameter.

Next, we can see the results achieved by a set of parameters combinations.

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import sklearn.metrics as metrics
from sklearn.ensemble import GradientBoostingClassifier
import ds_charts as ds

file_tag = 'diabetes'
filename = 'data/diabetes'
target = 'class'

train: pd.DataFrame = pd.read_csv(f'{filename}_train.csv')
trnY: np.ndarray = train.pop(target).values
trnX: np.ndarray = train.values
labels = pd.unique(trnY)

test: pd.DataFrame = pd.read_csv(f'{filename}_test.csv')
tstY: np.ndarray = test.pop(target).values
tstX: np.ndarray = test.values

n_estimators = [5, 10, 25, 50, 75, 100, 150, 200, 250, 300]
max_depths = [5, 10, 25]
learning_rate = [.1, .3, .5, .7, .9]
best = ('', 0, 0)
last_best = 0
best_tree = None

cols = len(max_depths)
plt.figure()
fig, axs = plt.subplots(1, cols, figsize=(cols*ds.HEIGHT, ds.HEIGHT), squeeze=False)
for k in range(len(max_depths)):
    d = max_depths[k]
    values = {}
    for lr in learning_rate:
        yvalues = []
        for n in n_estimators:
            gb = GradientBoostingClassifier(n_estimators=n, max_depth=d, learning_rate=lr)
            gb.fit(trnX, trnY)
            prdY = gb.predict(tstX)
            yvalues.append(metrics.accuracy_score(tstY, prdY))
            if yvalues[-1] > last_best:
                best = (d, lr, n)
                last_best = yvalues[-1]
                best_tree = gb
        values[lr] = yvalues
    ds.multiple_line_chart(n_estimators, values, ax=axs[0, k], title=f'Gradient Boorsting with max_depth={d}',
                           xlabel='nr estimators', ylabel='accuracy', percentage=True)
plt.savefig(f'images/{file_tag}_gb_study.png')
plt.show()
print('Best results with depth=%d, learning rate=%1.2f and %d estimators, with accuracy=%1.2f'%(best[0], best[1], best[2], last_best))

<Figure size 600x450 with 0 Axes>

Best results with depth=5, learning rate=0.10 and 75 estimators, with accuracy=0.77

After the plot you can see the parameters for which the best results were achieved. So let's see its performance, in that context in terms of other metrics.

prd_trn = best_tree.predict(trnX)
prd_tst = best_tree.predict(tstX)
ds.plot_evaluation_results(labels, trnY, prd_trn, tstY, prd_tst)
plt.savefig('images/{file_tag}_gb_best.png')
plt.show()