Eval Metric directionality? about shap-hypetune HOT 11 CLOSED

    greater_is_better : bool, default=False
        Effective only when hyperparameters searching.
        Whether the quantity to monitor is a score function,
        meaning high is good, or a loss function, meaning low is good.

from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification
from sklearn.metrics import brier_score_loss

from shaphypetune import BoostRFE

from lightgbm import *

X, y = make_classification(n_samples=6000, n_features=20, n_classes=2, 
                                   n_informative=4, n_redundant=6, random_state=0)

X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, shuffle=False)

def BRIER(y_true, y_hat):
    return 'brier', brier_score_loss(y_true, y_hat, pos_label=1), False

param_grid = {
    'learning_rate': [0.2, 0.1],
    'num_leaves': [25, 35],
    'max_depth': [10, 12]
}

model = BoostRFE(
    LGBMClassifier(n_estimators=150, random_state=0, metric="custom"), 
    param_grid=param_grid, min_features_to_select=1, step=1,
    greater_is_better=False
)
model.fit(
    X_train, y_train, 
    eval_set=[(X_valid, y_valid)], early_stopping_rounds=6, verbose=1, 
    eval_metric=BRIER
)

clf_xgb = XGBClassifier(n_estimators=2000,
                        random_state=0,
                        verbosity=3,
                        n_jobs=-1,
                        scale_pos_weight=1,
                        use_label_encoder=False,
                        objective='binary:logistic',
                        eval_set=[(cv_x, cv_y)])

    clf_xgb.fit(train_x, train_y)

class_pred = clf_xgb.predict(train_x)

balanced_accuracy = balanced_accuracy_score(class_pred, train_y)

brier_score = brier_score_loss(class_pred, train_y)

print(brier_score)

print(balanced_accuracy)

model = BoostRFE(clf_xgb, param_grid=param_dist, min_features_to_select=1, step=1, n_iter=8, sampling_seed=0)

model.fit(train_x, train_y, eval_set=[(cv_x, cv_y)], early_stopping_rounds=6, verbose=100,eval_metric=ACC)
print(model.estimator_, model.best_params_, model.best_score_, model.n_features_)

print(f"feature ranking {model.ranking_}")

model_ranking_list = list(model.ranking_)

print(model_ranking_list)

clf_xgb = XGBClassifier(n_estimators=2000,
                        random_state=0,
                        verbosity=3,
                        n_jobs=-1,
                        scale_pos_weight=1,
                        use_label_encoder=False,
                        objective='binary:logistic',
                        eval_set=[(cv_x, cv_y)],
                        eval_metric=ACC)

from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification
from sklearn.metrics import balanced_accuracy_score

from shaphypetune import BoostRFE

from xgboost import *

X, y = make_classification(n_samples=6000, n_features=20, n_classes=2, 
                                   n_informative=4, n_redundant=6, random_state=0)

X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, shuffle=False)

def ACC(y_pred, dtrain):
    y_true = dtrain.get_label()
    y_pred = (y_pred > 0.5).astype(int)
    err = 1 - balanced_accuracy_score(y_true, y_pred)
    return 'bal_acc', err

param_grid = {
    'learning_rate': [0.2, 0.1],
    'num_leaves': [25, 35],
    'max_depth': [10, 12]
}

model = BoostRFE(
    XGBClassifier(n_estimators=150, random_state=0, metric="custom"), 
    param_grid=param_grid, min_features_to_select=1, step=1,
    greater_is_better=False
)
model.fit(
    X_train, y_train, 
    eval_set=[(X_valid, y_valid)], early_stopping_rounds=6, verbose=1, 
    eval_metric=ACC
)