Reviewing a security-ML notebook

# ===================== PART B PROBE =====================
from sklearn.metrics import roc_auc_score
import numpy as np, pandas as pd
import xgboost as xgb

def sf_grade(a):
    return 'F' if a>=0.999 else 'D' if a>=0.99 else 'C' if a>=0.95 else 'B' if a>=0.85 else 'A'

def best_single(frame, labels):
    au = {}
    for c in feat:
        col = frame[c].to_numpy(float)
        if col.std() == 0: continue
        a = roc_auc_score(labels, col); au[c] = max(a, 1-a)
    return max(au.values()), max(au, key=au.get)

# B1 - does the grade survive a different random draw?
for seed in range(1, 9):
    s = X.sample(min(60_000, len(X)), random_state=seed)
    b, col = best_single(s, y[s.index])
    print(f"B1 seed {seed}: {b:.4f}  grade {sf_grade(b)}  ({col})")

# B2 - what is the value with no subsampling at all?
b, col = best_single(X, y)
print(f"\nB2 full corpus: {b:.4f}  grade {sf_grade(b)}  ({col})")

# B3 - does contamination depend on WHICH test rows were checked?
_trk = set(map(tuple, np.round(Xtr.to_numpy(), 6)))
_te  = np.round(Xte.to_numpy(), 6)
first = float(np.mean([tuple(r) in _trk for r in _te[:50_000]]))
ridx  = np.random.default_rng(0).choice(len(_te), size=min(50_000, len(_te)), replace=False)
rand  = float(np.mean([tuple(_te[i]) in _trk for i in ridx]))
print(f"B3 contamination: first-50k {first:.4f} vs random-50k {rand:.4f}")

# B4 - how many groups did the per-group report leave out?
_p = pd.Series(df['family'].to_numpy()[Xte.index][yte == 1]).value_counts()
print(f"B4 groups: {len(_p)} with >=1 positive, {int((_p<5).sum())} skipped by the count-5 floor "
      f"{list(_p[_p<5].index)}")

# B5 - the notebook drops ONE feature. Drop the top TWO and refit.
def probe_model():
    return xgb.XGBClassifier(n_estimators=120, max_depth=6, n_jobs=-1,
                             eval_metric='logloss', random_state=0)
m = probe_model().fit(Xtr, ytr)
base = roc_auc_score(yte, m.predict_proba(Xte)[:, 1])
imp = pd.Series(m.feature_importances_, index=feat).sort_values(ascending=False)
t1, t2 = imp.index[0], imp.index[1]
a1 = roc_auc_score(yte, probe_model().fit(Xtr.drop(columns=[t1]), ytr)
                        .predict_proba(Xte.drop(columns=[t1]))[:, 1])
a2 = roc_auc_score(yte, probe_model().fit(Xtr.drop(columns=[t1, t2]), ytr)
                        .predict_proba(Xte.drop(columns=[t1, t2]))[:, 1])
print(f"B5 ablation: base {base:.4f} | drop {t1} -> {a1:.4f} | drop {t1}+{t2} -> {a2:.4f}")