from ngboost import NGBClassifier, NGBRegressor
from ngboost.distns import k_categorical, Normal
from ngboost.scores import LogScore
from sklearn.datasets import load_breast_cancer, load_diabetes
from sklearn.model_selection import train_test_split
X, Y = load_diabetes(return_X_y=True)
X_reg_train, X_reg_test, Y_reg_train, Y_reg_test = train_test_split(X, Y, test_size=0.2)
X, y = load_breast_cancer(return_X_y=True)
y[0:15] = 2 # artificially make this a 3-class problem instead of a 2-class problem
X_cls_train, X_cls_test, Y_cls_train, Y_cls_test = train_test_split(X, y, test_size=0.2)Staged Prediction¶
All fit NGBoost objects support staged prediction.
ngb_cls = NGBClassifier(Dist=k_categorical(3), Score=LogScore, n_estimators=500, verbose=False).fit(X_cls_train, Y_cls_train)For instance, to get the predictions on the first 5 examples after fitting 415 base learners, use:
preds = ngb_cls.staged_predict(X_cls_test)
preds[415][0:5]array([1, 1, 0, 1, 0])pred_dists = ngb_cls.staged_pred_dist(X_cls_test)
pred_dists[415][0:5].params{'p0': array([0.00201258, 0.00201258, 0.52907719, 0.08351746, 0.97426859]),
'p1': array([0.99785485, 0.99785485, 0.46686649, 0.91584223, 0.02395252]),
'p2': array([0.00013257, 0.00013257, 0.00405632, 0.00064031, 0.00177889])}This is useful in conjunction with tracking errors on a validation set, which you can do by passing the X_val and Y_val arguments and then inspecting the .best_val_loss_itr instance attribute.
ngb = NGBRegressor()
ngb.fit(X_reg_train, Y_reg_train, X_val=X_reg_test, Y_val=Y_reg_test) # use a validation set instead of test set here in your own work
print(ngb.best_val_loss_itr)
best_preds = ngb.predict(X_reg_test, max_iter=ngb.best_val_loss_itr)[iter 0] loss=5.7429 val_loss=5.8408 scale=1.0000 norm=64.6711
[iter 100] loss=5.3637 val_loss=5.6704 scale=2.0000 norm=85.8650
[iter 200] loss=5.0970 val_loss=5.8551 scale=1.0000 norm=33.6059
[iter 300] loss=4.9509 val_loss=6.2818 scale=1.0000 norm=30.4502
[iter 400] loss=4.8622 val_loss=6.7078 scale=1.0000 norm=28.7510
100
Early Stopping¶
NGBoost also has early stopping. If an integer early_stopping_rounds and a validation set (X_val,Y_val) are passed to fit(), the algorithm will stop running after the validation loss has increased for early_stopping_rounds of consecutive iterations.
_ = NGBRegressor().fit(X_reg_train, Y_reg_train, X_val=X_reg_test, Y_val=Y_reg_test, early_stopping_rounds=2)[iter 0] loss=5.7429 val_loss=5.8408 scale=1.0000 norm=64.6711
== Early stopping achieved.
== Best iteration / VAL91 (val_loss=5.6711)
Validation set sample weights can be passed using the val_sample_weight argument to fit.
Using sklearn Model Selection¶
sklearn methods are compatible with NGBoost.
from sklearn.model_selection import GridSearchCV
from sklearn.tree import DecisionTreeRegressor
b1 = DecisionTreeRegressor(criterion='friedman_mse', max_depth=2)
b2 = DecisionTreeRegressor(criterion='friedman_mse', max_depth=4)
param_grid = {
'minibatch_frac': [1.0, 0.5],
'Base': [b1, b2]
}
ngb = NGBRegressor(Dist=Normal, verbose=False)
grid_search = GridSearchCV(ngb, param_grid=param_grid, cv=5)
grid_search.fit(X_reg_train, Y_reg_train)
print(grid_search.best_params_){'Base': DecisionTreeRegressor(criterion='friedman_mse', max_depth=2), 'minibatch_frac': 0.5}