Feature Importance Caveats

# Feature evidence on training data only
# Step 1 — Import the module so its functions/classes are available to the rest of this example.
import numpy as np
# Step 2 — Import only the named objects needed by the following steps, keeping dependencies explicit.
from scipy.stats import ttest_ind
# Step 3 — Import only the named objects needed by the following steps, keeping dependencies explicit.
from sklearn.metrics import roc_auc_score
# Step 4 — Import only the named objects needed by the following steps, keeping dependencies explicit.
from sklearn.linear_model import LogisticRegression
# Step 5 — Import only the named objects needed by the following steps, keeping dependencies explicit.
from sklearn.feature_selection import RFE
# Step 6 — Import only the named objects needed by the following steps, keeping dependencies explicit.
from sklearn.inspection import permutation_importance

# Step 7 — Compute the right-hand expression and store its result in `feature_rows` for the next step.
feature_rows = []
# Step 8 — Iterate through the collection so the indented block is applied once for each item.
for name in feature_names:
    # Step 9 — Execute this statement and inspect how it changes the current value, object or program state.
    x0 = X_train.loc[y_train == 0, name].dropna()
    # Step 10 — Execute this statement and inspect how it changes the current value, object or program state.
    x1 = X_train.loc[y_train == 1, name].dropna()

    # 1) Statistical significance: unequal-variance two-sample test
    # Step 11 — Compute the right-hand expression and store its result in `t_stat, p_value` for the next step.
    t_stat, p_value = ttest_ind(x0, x1, equal_var=False)

    # 2) Univariate discrimination; direction-adjusted for importance
    # Step 12 — Compute the right-hand expression and store its result in `auc` for the next step.
    auc = roc_auc_score(y_train, X_train[name])
    # Step 13 — Compute the right-hand expression and store its result in `auc_importance` for the next step.
    auc_importance = max(auc, 1 - auc)

    # Step 14 — Execute this statement and inspect how it changes the current value, object or program state.
    feature_rows.append((name, t_stat, p_value, auc_importance))

# 3) Model-based subset selection: fit only on development data
# Step 15 — Instantiate `base` with the chosen algorithm/configuration before fitting it to data.
base = LogisticRegression(max_iter=2000)
# Step 16 — Compute the right-hand expression and store its result in `rfe` for the next step.
rfe = RFE(base, n_features_to_select=3)
# Step 17 — Fit the model or transformer, learning its parameters from the supplied training data.
rfe.fit(X_train_scaled, y_train)
# Step 18 — Construct `selected` as an array so vectorised numerical operations can be applied consistently.
selected = np.array(feature_names)[rfe.support_]

# 4) Model reliance: evaluate permutation loss on validation data
# Step 19 — Fit the model or transformer, learning its parameters from the supplied training data.
model = LogisticRegression(max_iter=2000).fit(
    X_train_scaled[:, rfe.support_], y_train
)
# Step 20 — Compute the right-hand expression and store its result in `perm` for the next step.
perm = permutation_importance(
    model,
    X_valid_scaled[:, rfe.support_],
    y_valid,
    scoring="roc_auc",
    n_repeats=20,
    random_state=42,
)

# Step 21 — Display the current value explicitly so the result/state can be inspected during execution.
print("Selected by RFE:", selected.tolist())
# Step 22 — Display the current value explicitly so the result/state can be inspected during execution.
print("Validation permutation ΔAUC:", perm.importances_mean.round(3))