"""Spectral Passport: cross-instrument conformal coverage on the eight-spectrometer Raman collection. Requires numpy, scipy, pandas, pyarrow, scikit-learn. Writes results.csv.""" import os, numpy as np, pandas as pd from scipy.signal import savgol_filter from sklearn.cross_decomposition import PLSRegression from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_predict, StratifiedKFold, GroupKFold from sklearn.metrics import roc_auc_score INS = ['Anton532', 'Anton785', 'Kaiser', 'Metrohm', 'MettlerToledo', 'Tec5', 'Timegate', 'Tornado'] ANA = ['Glucose', 'Acetate', 'MagnesiumSulfate'] GRID = np.arange(400, 1800.1, 2.0) ALPHA = 0.10 REPS = 20 rng0 = np.random.default_rng(20260910) DATA_DIR = os.environ.get('RAMAN_DIR', 'data') HF = 'https://huggingface.co/datasets/chlange/RamanSpectraBioprocessAnalystes{}/resolve/main/{}.parquet' def load(inst, split): # Reads data/_.parquet if present, otherwise downloads it from the Hugging Face Hub # (files train.parquet, val.parquet and test.parquet, checked 13 November 2026). p = os.path.join(DATA_DIR, f'{inst}_{split}.parquet') if not os.path.exists(p): os.makedirs(DATA_DIR, exist_ok=True) pd.read_parquet(HF.format(inst, split)).to_parquet(p) return pd.read_parquet(p) def mix_id(df): # a mixture is identified by its three reference concentrations; replicate spectra share it return np.array(['|'.join(f'{v:.3f}' for v in r) for r in df[ANA].values]) def prep(df): wn = np.array([float(c) for c in df.columns[:-3]]) X = df.iloc[:, :-3].values.astype(float) Xi = np.vstack([np.interp(GRID, wn, x) for x in X]) Xd = savgol_filter(Xi, 15, 2, deriv=1, axis=1) Xd = (Xd - Xd.mean(1, keepdims=True)) / Xd.std(1, keepdims=True) return Xd, df[ANA].values.astype(float), mix_id(df) D = {} for i in INS: D[i] = {s: prep(load(i, s)) for s in ['train', 'val', 'test']} def one_per_mixture(groups, rng): # indices of one randomly chosen replicate spectrum for each mixture return np.array([rng.choice(np.where(groups == g)[0]) for g in np.unique(groups)]) def qhat(scores, alpha=ALPHA): n = len(scores) k = int(np.ceil((n + 1) * (1 - alpha))) return np.inf if k > n else np.sort(scores)[k - 1] def wqhat(scores, w_cal, w_test, alpha=ALPHA): # Tibshirani et al. (2019): weighted quantile with mass w_test at +inf o = np.argsort(scores); s = scores[o]; w = w_cal[o] tot = w.sum() + w_test c = np.cumsum(w) / tot idx = np.searchsorted(c, 1 - alpha) return np.inf if idx >= len(s) else s[idx] models = {} for i in INS: Xtr, Ytr, Gtr = D[i]['train']; Xva, Yva, _ = D[i]['val'] for a, an in enumerate(ANA): # number of components chosen by 5-fold cross-validation within the training set, # with folds grouped by mixture; the validation set is kept for calibration only best = None for nc in range(1, 16): pr = np.zeros(len(Ytr)) for tr, te in GroupKFold(5).split(Xtr, Ytr[:, a], Gtr): pr[te] = PLSRegression(nc, scale=False).fit(Xtr[tr], Ytr[tr, a]).predict(Xtr[te]).ravel() r = np.sqrt(np.mean((pr - Ytr[:, a]) ** 2)) if best is None or r < best[0]: best = (r, nc) m = PLSRegression(best[1], scale=False).fit(Xtr, Ytr[:, a]) rv = np.sqrt(np.mean((m.predict(Xva).ravel() - Yva[:, a]) ** 2)) models[(i, an)] = (rv, best[1], m) rows = [] for s in INS: Xs = np.vstack([D[s]['train'][0], D[s]['val'][0]]) for t in INS: Xt_unl = D[t]['train'][0] Xt_te, Yt_te, Gt_te = D[t]['test'] Xs_cal, Ys_cal, Gs_cal = D[s]['val'] # domain classifier for shift diagnosis and weights if s != t: Z = np.vstack([Xs, Xt_unl]); lab = np.r_[np.zeros(len(Xs)), np.ones(len(Xt_unl))] pca = PCA(20, random_state=0).fit(Z) Zp = pca.transform(Z) clf = LogisticRegression(C=0.1, max_iter=5000) p = cross_val_predict(clf, Zp, lab, cv=StratifiedKFold(5, shuffle=True, random_state=0), method='predict_proba')[:, 1] auc = roc_auc_score(lab, p) clf.fit(Zp, lab) def wfun(X): pp = np.clip(clf.predict_proba(pca.transform(X))[:, 1], 1e-12, 1 - 1e-12) return pp / (1 - pp) * len(Xs) / len(Xt_unl) else: auc = 0.5 def wfun(X): return np.ones(len(X)) wc_all = wfun(Xs_cal); wte = wfun(Xt_te) draws = [one_per_mixture(Gs_cal, np.random.default_rng(1000 + r)) for r in range(REPS)] wc = wc_all[draws[0]] ess = wc.sum() ** 2 / (wc ** 2).sum() for a, an in enumerate(ANA): rmse_src, nc, m = models[(s, an)] yhat_te = m.predict(Xt_te).ravel() res_te = np.abs(yhat_te - Yt_te[:, a]) # calibration uses one replicate per mixture, so that calibration scores are exchangeable # across mixtures; results are averaged over REPS random choices of replicate res_cal_all = np.abs(m.predict(Xs_cal).ravel() - Ys_cal[:, a]) qd = [qhat(res_cal_all[ix]) for ix in draws] cov_split = float(np.mean([np.mean(res_te <= q_) for q_ in qd])) q = float(np.median(qd)) sc_cal = res_cal_all[draws[0]] n_mix_te = len(np.unique(Gt_te)) qs = np.array([wqhat(sc_cal, wc, w) for w in wte]) cov_w = float(np.mean(res_te <= qs)); finite_w = float(np.mean(np.isfinite(qs))) width_w = float(np.median(2 * qs[np.isfinite(qs)])) if np.isfinite(qs).any() else np.inf rec = {} Xt_tr, Yt_tr, Gt_tr = D[t]['train'] ug = np.unique(Gt_tr) for k in (10, 20): covs, wids = [], [] for r in range(REPS): # k labelled spectra from k distinct target mixtures, one replicate each gs = rng0.choice(ug, k, replace=False) idx = np.array([rng0.choice(np.where(Gt_tr == g)[0]) for g in gs]) sck = np.abs(m.predict(Xt_tr[idx]).ravel() - Yt_tr[idx, a]) qk = qhat(sck) covs.append(np.mean(res_te <= qk)); wids.append(2 * qk) rec[k] = (float(np.mean(covs)), float(np.median(wids))) rows.append(dict(source=s, target=t, analyte=an, ncomp=nc, rmse_src_val=float(rmse_src), n_mix_cal=len(sc_cal), n_mix_test=n_mix_te, rmse_target=float(np.sqrt(np.mean((yhat_te - Yt_te[:, a]) ** 2))), range_target=float(np.ptp(Yt_te[:, a])), auc=float(auc), ess=float(ess), n_cal=len(sc_cal), cov_split=cov_split, width_split=float(2 * q), cov_weighted=cov_w, finite_weighted=finite_w, width_weighted=width_w, cov_recal10=rec[10][0], width_recal10=rec[10][1], cov_recal20=rec[20][0], width_recal20=rec[20][1])) df = pd.DataFrame(rows) df.to_csv('results.csv', index=False) print(df.groupby(df.source == df.target)[['cov_split', 'cov_weighted', 'cov_recal10', 'cov_recal20', 'auc', 'ess']].describe().T.round(3).to_string())