lightgbm原始函数与Sklearn

LightGBM

1.原生态的LightGBM

https://github.com/microsoft/LightGBM/blob/master/examples/python-guide/simple_example.py

# coding: utf-8
import json
import lightgbm as lgb
import pandas as pd
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')

# 加载数据集合
print('Load data...')
df_train = pd.read_csv('I://机器学习\七月在线-金融风控实战/资料/第六课资料/regression.train.txt', header=None, sep='\t')
df_test = pd.read_csv('I://机器学习/七月在线-金融风控实战/资料/第六课资料/regression.test.txt', header=None, sep='\t')

Load data...

print(df_train.shape)
df_train.head()

(7000, 29)

	0	1	2	3	4	5	6	7	8	9	...	19	20	21	22	23	24	25	26	27	28
0	1	0.869	-0.635	0.226	0.327	-0.690	0.754	-0.249	-1.092	0.000	...	-0.010	-0.046	3.102	1.354	0.980	0.978	0.920	0.722	0.989	0.877
1	1	0.908	0.329	0.359	1.498	-0.313	1.096	-0.558	-1.588	2.173	...	-1.139	-0.001	0.000	0.302	0.833	0.986	0.978	0.780	0.992	0.798
2	1	0.799	1.471	-1.636	0.454	0.426	1.105	1.282	1.382	0.000	...	1.129	0.900	0.000	0.910	1.108	0.986	0.951	0.803	0.866	0.780
3	0	1.344	-0.877	0.936	1.992	0.882	1.786	-1.647	-0.942	0.000	...	-0.678	-1.360	0.000	0.947	1.029	0.999	0.728	0.869	1.027	0.958
4	1	1.105	0.321	1.522	0.883	-1.205	0.681	-1.070	-0.922	0.000	...	-0.374	0.113	0.000	0.756	1.361	0.987	0.838	1.133	0.872	0.808

5 rows × 29 columns

print(df_test.shape)
df_test.head()

(500, 29)

	0	1	2	3	4	5	6	7	8	9	...	19	20	21	22	23	24	25	26	27	28
0	1	0.644	0.247	-0.447	0.862	0.374	0.854	-1.126	-0.790	2.173	...	-0.190	-0.744	3.102	0.958	1.061	0.980	0.875	0.581	0.905	0.796
1	0	0.385	1.800	1.037	1.044	0.349	1.502	-0.966	1.734	0.000	...	-0.440	0.638	3.102	0.695	0.909	0.981	0.803	0.813	1.149	1.116
2	0	1.214	-0.166	0.004	0.505	1.434	0.628	-1.174	-1.230	1.087	...	-1.383	1.355	0.000	0.848	0.911	1.043	0.931	1.058	0.744	0.696
3	1	0.420	1.111	0.137	1.516	-1.657	0.854	0.623	1.605	1.087	...	0.731	1.424	3.102	1.597	1.282	1.105	0.730	0.148	1.231	1.234
4	0	0.897	-1.703	-1.306	1.022	-0.729	0.836	0.859	-0.333	2.173	...	-2.019	-0.289	0.000	0.805	0.930	0.984	1.430	2.198	1.934	1.684

5 rows × 29 columns

# 设定训练集和测试集
y_train = df_train[0].values
y_test = df_test[0].values
X_train = df_train.drop(0, axis=1).values
X_test = df_test.drop(0, axis=1).values

# 构建lgb中的Dataset格式，和xgboost中的DMatrix是对应的
lgb_train = lgb.Dataset(X_train, y_train)
lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)# If this is Dataset for validation, training data should be used as reference.

#help(lgb.Dataset)

#help(lgb.train.params)

# 敲定好一组参数
params = {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'regression',
    'metric': {'l2', 'auc'},
    'num_leaves': 31,
    'learning_rate': 0.05,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.8,
    'bagging_freq': 5,
    'verbose': 0
}

print('开始训练...')
# 训练
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=30,# default=100,Number of boosting iterations. 训练次数
                valid_sets=lgb_eval,
                #valid_sets=[lgb_train,lgb_eval],
                early_stopping_rounds=5)

# 保存模型
print('保存模型...')
# 保存模型到文件中
gbm.save_model('model.txt')

开始训练...
[1]	valid_0's auc: 0.721096	valid_0's l2: 0.243898
Training until validation scores don't improve for 5 rounds.
[2]	valid_0's auc: 0.732932	valid_0's l2: 0.240605
[3]	valid_0's auc: 0.773583	valid_0's l2: 0.236472
[4]	valid_0's auc: 0.781089	valid_0's l2: 0.232586
[5]	valid_0's auc: 0.787902	valid_0's l2: 0.22865
[6]	valid_0's auc: 0.780565	valid_0's l2: 0.226187
[7]	valid_0's auc: 0.786571	valid_0's l2: 0.223738
[8]	valid_0's auc: 0.784918	valid_0's l2: 0.221012
[9]	valid_0's auc: 0.784878	valid_0's l2: 0.218429
[10]	valid_0's auc: 0.788917	valid_0's l2: 0.215505
[11]	valid_0's auc: 0.791586	valid_0's l2: 0.213027
[12]	valid_0's auc: 0.793884	valid_0's l2: 0.210809
[13]	valid_0's auc: 0.796174	valid_0's l2: 0.208612
[14]	valid_0's auc: 0.793731	valid_0's l2: 0.207468
[15]	valid_0's auc: 0.794247	valid_0's l2: 0.206009
[16]	valid_0's auc: 0.793715	valid_0's l2: 0.20465
[17]	valid_0's auc: 0.797085	valid_0's l2: 0.202489
[18]	valid_0's auc: 0.802615	valid_0's l2: 0.200668
[19]	valid_0's auc: 0.802793	valid_0's l2: 0.19925
[20]	valid_0's auc: 0.802615	valid_0's l2: 0.198136
[21]	valid_0's auc: 0.80139	valid_0's l2: 0.197207
[22]	valid_0's auc: 0.799237	valid_0's l2: 0.196629
[23]	valid_0's auc: 0.800044	valid_0's l2: 0.195623
[24]	valid_0's auc: 0.800269	valid_0's l2: 0.194751
Early stopping, best iteration is:
[19]	valid_0's auc: 0.802793	valid_0's l2: 0.19925
保存模型...





<lightgbm.basic.Booster at 0x14766400>

gbm.best_iteration #最好的迭代次数

19

print('开始预测...')
y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
# 评估
print('预估结果的rmse为:')
print(mean_squared_error(y_test, y_pred) ** 0.5)

开始预测...
预估结果的rmse为:
0.44637392507504314

2.添加样本权重训练

import json
import lightgbm as lgb
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings("ignore")

# 加载数据集
print('加载数据...')
df_train = pd.read_csv('I://机器学习\七月在线-金融风控实战/资料/第六课资料/binary.test/binary.train', header=None, sep='\t')
df_test = pd.read_csv('I://机器学习\七月在线-金融风控实战/资料/第六课资料/binary.test/binary.test', header=None, sep='\t')
W_train = pd.read_csv('I://机器学习\七月在线-金融风控实战/资料/第六课资料/binary.test/binary.train.weight', header=None)[0]
W_test = pd.read_csv('I://机器学习\七月在线-金融风控实战/资料/第六课资料/binary.test/binary.test.weight', header=None)[0]

加载数据...

print(df_train.shape)
df_train.head()

(7000, 29)

	0	1	2	3	4	5	6	7	8	9	...	19	20	21	22	23	24	25	26	27	28
0	1	0.869	-0.635	0.226	0.327	-0.690	0.754	-0.249	-1.092	0.000	...	-0.010	-0.046	3.102	1.354	0.980	0.978	0.920	0.722	0.989	0.877
1	1	0.908	0.329	0.359	1.498	-0.313	1.096	-0.558	-1.588	2.173	...	-1.139	-0.001	0.000	0.302	0.833	0.986	0.978	0.780	0.992	0.798
2	1	0.799	1.471	-1.636	0.454	0.426	1.105	1.282	1.382	0.000	...	1.129	0.900	0.000	0.910	1.108	0.986	0.951	0.803	0.866	0.780
3	0	1.344	-0.877	0.936	1.992	0.882	1.786	-1.647	-0.942	0.000	...	-0.678	-1.360	0.000	0.947	1.029	0.999	0.728	0.869	1.027	0.958
4	1	1.105	0.321	1.522	0.883	-1.205	0.681	-1.070	-0.922	0.000	...	-0.374	0.113	0.000	0.756	1.361	0.987	0.838	1.133	0.872	0.808

5 rows × 29 columns

print(W_train.shape)
W_train.head()

(7000,)





0    1.2
1    1.1
2    1.0
3    1.0
4    1.0
Name: 0, dtype: float64

y_train = df_train[0].values
y_test = df_test[0].values
X_train = df_train.drop(0, axis=1).values
X_test = df_test.drop(0, axis=1).values

num_train, num_feature = X_train.shape

# 加载数据的同时加载权重
lgb_train = lgb.Dataset(X_train, y_train,weight=W_train, free_raw_data=False)
lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train,weight=W_test, free_raw_data=False)

# 设定参数
params = {
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'metric': 'binary_logloss',
    'num_leaves': 31,
    'learning_rate': 0.05,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.8,
    'bagging_freq': 5,
    'verbose': 0
}

# 产出特征名称
feature_name = ['feature_' + str(col) for col in range(num_feature)]

print('开始训练...')
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=10,
                valid_sets=lgb_train,  # 评估训练集
                #valid_sets=[lgb_train,lgb_eval],
                feature_name=feature_name,
                categorical_feature=[21])

开始训练...
[1]	training's binary_logloss: 0.680539
[2]	training's binary_logloss: 0.672148
[3]	training's binary_logloss: 0.6646
[4]	training's binary_logloss: 0.655873
[5]	training's binary_logloss: 0.647672
[6]	training's binary_logloss: 0.640947
[7]	training's binary_logloss: 0.635104
[8]	training's binary_logloss: 0.628453
[9]	training's binary_logloss: 0.622051
[10]	training's binary_logloss: 0.616221

3.模型的载入与预测

# 查看特征名称
print('完成10轮训练...')
print('第7个特征为:')
print(repr(lgb_train.feature_name[6]))

# 存储模型
gbm.save_model('./lgb_model.txt')

# 特征名称
print('特征名称:')
print(gbm.feature_name())

# 特征重要度
print('特征重要度:')
print(list(gbm.feature_importance()))

# 加载模型
print('加载模型用于预测')
bst = lgb.Booster(model_file='./lgb_model.txt')
# 预测
y_pred = bst.predict(X_test)
# 在测试集评估效果
print('在测试集上的rmse为:')
print(mean_squared_error(y_test, y_pred) ** 0.5)

完成10轮训练...
第7个特征为:
'feature_6'
特征名称:
['feature_0', 'feature_1', 'feature_2', 'feature_3', 'feature_4', 'feature_5', 'feature_6', 'feature_7', 'feature_8', 'feature_9', 'feature_10', 'feature_11', 'feature_12', 'feature_13', 'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18', 'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23', 'feature_24', 'feature_25', 'feature_26', 'feature_27']
特征重要度:
[14, 4, 6, 19, 1, 32, 1, 1, 1, 17, 1, 6, 0, 10, 2, 2, 0, 3, 0, 3, 0, 0, 33, 2, 35, 43, 30, 34]
加载模型用于预测
在测试集上的rmse为:
0.46456635981135486

4.接着之前的模型继续训练

# 继续训练
# 从./model/model.txt中加载模型初始化
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=10,
                init_model='./lgb_model.txt',
                valid_sets=lgb_eval)

print('以旧模型为初始化，完成第 10-20 轮训练...')

# 在训练的过程中调整超参数
# 比如这里调整的是学习率
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=10,
                init_model=gbm,
                learning_rates=lambda iter: 0.05 * (0.99 ** iter),
                valid_sets=lgb_eval)

print('逐步调整学习率完成第 20-30 轮训练...')

# 调整其他超参数
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=10,
                init_model=gbm,
                valid_sets=lgb_eval,
                callbacks=[lgb.reset_parameter(bagging_fraction=[0.7] * 5 + [0.6] * 5)])

print('逐步调整bagging比率完成第 30-40 轮训练...')

[11]	valid_0's binary_logloss: 0.617219
[12]	valid_0's binary_logloss: 0.613898
[13]	valid_0's binary_logloss: 0.610079
[14]	valid_0's binary_logloss: 0.606362
[15]	valid_0's binary_logloss: 0.602833
[16]	valid_0's binary_logloss: 0.599507
[17]	valid_0's binary_logloss: 0.596463
[18]	valid_0's binary_logloss: 0.592853
[19]	valid_0's binary_logloss: 0.589744
[20]	valid_0's binary_logloss: 0.586852
以旧模型为初始化，完成第 10-20 轮训练...
[21]	valid_0's binary_logloss: 0.617219
[22]	valid_0's binary_logloss: 0.61393
[23]	valid_0's binary_logloss: 0.610107
[24]	valid_0's binary_logloss: 0.607164
[25]	valid_0's binary_logloss: 0.603978
[26]	valid_0's binary_logloss: 0.600968
[27]	valid_0's binary_logloss: 0.598118
[28]	valid_0's binary_logloss: 0.594547
[29]	valid_0's binary_logloss: 0.591645
[30]	valid_0's binary_logloss: 0.588981
逐步调整学习率完成第 20-30 轮训练...
[31]	valid_0's binary_logloss: 0.618755
[32]	valid_0's binary_logloss: 0.614889
[33]	valid_0's binary_logloss: 0.611353
[34]	valid_0's binary_logloss: 0.60686
[35]	valid_0's binary_logloss: 0.603673
[36]	valid_0's binary_logloss: 0.601463
[37]	valid_0's binary_logloss: 0.598129
[38]	valid_0's binary_logloss: 0.595154
[39]	valid_0's binary_logloss: 0.592258
[40]	valid_0's binary_logloss: 0.589495
逐步调整bagging比率完成第 30-40 轮训练...

5.自定义损失函数

# 类似在xgboost中的形式
# 自定义损失函数需要
def loglikelood(preds, train_data):
    labels = train_data.get_label()
    preds = 1. / (1. + np.exp(-preds))
    grad = preds - labels
    hess = preds * (1. - preds)
    return grad, hess


# 自定义评估函数
def binary_error(preds, train_data):
    labels = train_data.get_label()
    return 'error', np.mean(labels != (preds > 0.5)), False


gbm = lgb.train(params,
                lgb_train,
                num_boost_round=10,
                init_model=gbm,
                fobj=loglikelood,
                feval=binary_error,
                valid_sets=lgb_eval)

print('用自定义的损失函数与评估标准完成第40-50轮...')

[41]	valid_0's binary_logloss: 4.79457	valid_0's error: 0.398
[42]	valid_0's binary_logloss: 4.957	valid_0's error: 0.4
[43]	valid_0's binary_logloss: 4.81505	valid_0's error: 0.392
[44]	valid_0's binary_logloss: 5.34566	valid_0's error: 0.382
[45]	valid_0's binary_logloss: 5.27353	valid_0's error: 0.376
[46]	valid_0's binary_logloss: 5.32211	valid_0's error: 0.368
[47]	valid_0's binary_logloss: 5.19164	valid_0's error: 0.36
[48]	valid_0's binary_logloss: 5.36009	valid_0's error: 0.354
[49]	valid_0's binary_logloss: 5.46305	valid_0's error: 0.35
[50]	valid_0's binary_logloss: 5.51906	valid_0's error: 0.34
用自定义的损失函数与评估标准完成第40-50轮...

sklearn与LightGBM配合使用

1.LightGBM建模，sklearn评估

import lightgbm as lgb
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import GridSearchCV

# 加载数据
print('加载数据...')
df_train = pd.read_csv('I://机器学习/七月在线-金融风控实战/资料/第六课资料/regression.train.txt', header=None, sep='\t')
df_test = pd.read_csv('I://机器学习/七月在线-金融风控实战/资料/第六课资料/regression.test.txt', header=None, sep='\t')

# 取出特征和标签
y_train = df_train[0].values
y_test = df_test[0].values
X_train = df_train.drop(0, axis=1).values
X_test = df_test.drop(0, axis=1).values

加载数据...

print('开始训练...')
# 直接初始化LGBMRegressor
# 这个LightGBM的Regressor和sklearn中其他Regressor基本是一致的
gbm = lgb.LGBMRegressor(objective='regression',
                        num_leaves=31,
                        learning_rate=0.05,
                        n_estimators=20)

# 使用fit函数拟合
gbm.fit(X_train, y_train,
        eval_set=[(X_test, y_test)],
        eval_metric='l1',
        early_stopping_rounds=5)

# 预测
print('开始预测...')
y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration_)
# 评估预测结果
print('预测结果的rmse是:')
print(mean_squared_error(y_test, y_pred) ** 0.5)

开始训练...
[1]	valid_0's l2: 0.242763	valid_0's l1: 0.491735
Training until validation scores don't improve for 5 rounds.
[2]	valid_0's l2: 0.237895	valid_0's l1: 0.486563
[3]	valid_0's l2: 0.233277	valid_0's l1: 0.481489
[4]	valid_0's l2: 0.22925	valid_0's l1: 0.476848
[5]	valid_0's l2: 0.226155	valid_0's l1: 0.47305
[6]	valid_0's l2: 0.222963	valid_0's l1: 0.469049
[7]	valid_0's l2: 0.220364	valid_0's l1: 0.465556
[8]	valid_0's l2: 0.217872	valid_0's l1: 0.462208
[9]	valid_0's l2: 0.215328	valid_0's l1: 0.458676
[10]	valid_0's l2: 0.212743	valid_0's l1: 0.454998
[11]	valid_0's l2: 0.210805	valid_0's l1: 0.452047
[12]	valid_0's l2: 0.208945	valid_0's l1: 0.449158
[13]	valid_0's l2: 0.206986	valid_0's l1: 0.44608
[14]	valid_0's l2: 0.205513	valid_0's l1: 0.443554
[15]	valid_0's l2: 0.203728	valid_0's l1: 0.440643
[16]	valid_0's l2: 0.201865	valid_0's l1: 0.437687
[17]	valid_0's l2: 0.200639	valid_0's l1: 0.435454
[18]	valid_0's l2: 0.199522	valid_0's l1: 0.433288
[19]	valid_0's l2: 0.198552	valid_0's l1: 0.431297
[20]	valid_0's l2: 0.197238	valid_0's l1: 0.428946
Did not meet early stopping. Best iteration is:
[20]	valid_0's l2: 0.197238	valid_0's l1: 0.428946
开始预测...
预测结果的rmse是:
0.4441153344254208

2.网格搜索查找最优超参数

# 配合scikit-learn的网格搜索交叉验证选择最优超参数
estimator = lgb.LGBMRegressor(num_leaves=31)

param_grid = {
    'learning_rate': [0.01, 0.1, 1],
    'n_estimators': [20, 40]
}

gbm = GridSearchCV(estimator, param_grid)

gbm.fit(X_train, y_train)

print('用网格搜索找到的最优超参数为:')
print(gbm.best_params_)

用网格搜索找到的最优超参数为:
{'learning_rate': 0.1, 'n_estimators': 40}

3.绘图解释

# coding: utf-8
import lightgbm as lgb
import pandas as pd
import matplotlib.pyplot as plt

# 加载数据集
print('加载数据...')
df_train = pd.read_csv('I://机器学习/七月在线-金融风控实战/资料/第六课资料/regression.train.txt', header=None, sep='\t')
df_test = pd.read_csv('I://机器学习/七月在线-金融风控实战/资料/第六课资料/regression.test.txt', header=None, sep='\t')

加载数据...

# 取出特征和标签
y_train = df_train[0].values
y_test = df_test[0].values
X_train = df_train.drop(0, axis=1).values
X_test = df_test.drop(0, axis=1).values

# 构建lgb中的Dataset数据格式
lgb_train = lgb.Dataset(X_train, y_train)
lgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train)

# 设定参数
params = {
    'num_leaves': 5,
    'metric': ('l1', 'l2'),
    'verbose': 0
}

evals_result = {}  # to record eval results for plotting

print('开始训练...')
# 训练
gbm = lgb.train(params,
                lgb_train,
                num_boost_round=100,
                valid_sets=[lgb_train, lgb_test],
                feature_name=['f' + str(i + 1) for i in range(28)],
                categorical_feature=[21],
                evals_result=evals_result,
                verbose_eval=10)

print('在训练过程中绘图...')
ax = lgb.plot_metric(evals_result, metric='l1')
plt.show()

print('画出特征重要度...')
ax = lgb.plot_importance(gbm, max_num_features=10)
plt.show()

print('画出第84颗树...')
ax = lgb.plot_tree(gbm, tree_index=83, figsize=(20, 8), show_info=['split_gain'])
plt.show()

print('用graphviz画出第84颗树...')
graph = lgb.create_tree_digraph(gbm, tree_index=83, name='Tree84')
graph.render(view=True)

开始训练...
[10]	training's l1: 0.457448	training's l2: 0.217995	valid_1's l1: 0.456464	valid_1's l2: 0.21641
[20]	training's l1: 0.436869	training's l2: 0.205099	valid_1's l1: 0.434057	valid_1's l2: 0.201616
[30]	training's l1: 0.421302	training's l2: 0.197421	valid_1's l1: 0.417019	valid_1's l2: 0.192514
[40]	training's l1: 0.411107	training's l2: 0.192856	valid_1's l1: 0.406303	valid_1's l2: 0.187258
[50]	training's l1: 0.403695	training's l2: 0.189593	valid_1's l1: 0.398997	valid_1's l2: 0.183688
[60]	training's l1: 0.398704	training's l2: 0.187043	valid_1's l1: 0.393977	valid_1's l2: 0.181009
[70]	training's l1: 0.394876	training's l2: 0.184982	valid_1's l1: 0.389805	valid_1's l2: 0.178803
[80]	training's l1: 0.391147	training's l2: 0.1828	valid_1's l1: 0.386476	valid_1's l2: 0.176799
[90]	training's l1: 0.388101	training's l2: 0.180817	valid_1's l1: 0.384404	valid_1's l2: 0.175775
[100]	training's l1: 0.385174	training's l2: 0.179171	valid_1's l1: 0.382929	valid_1's l2: 0.175321
在训练过程中绘图...

画出特征重要度...

画出第84颗树...

用graphviz画出第84颗树…

'Tree84.gv.pdf'

`