{
"seed": 20261014,
"train": {
"rows": 894,
"late_rate": 0.23937360178970918,
"missing_inputs": {
"distance_km": 52,
"queued_jobs": 30,
"item_count": 0,
"weekday": 0,
"promised_hours": 0,
"priority": 0,
"zone": 0,
"depot_id": 0
}
},
"validation": {
"rows": 294,
"late_rate": 0.24149659863945577,
"missing_inputs": {
"distance_km": 16,
"queued_jobs": 3,
"item_count": 0,
"weekday": 0,
"promised_hours": 0,
"priority": 0,
"zone": 0,
"depot_id": 0
}
},
"calibration_rows": 144,
"policy_rows": 144,
"split_demo": {
"random": {
"train_rows": 715,
"holdout_rows": 179,
"overlapping_depots": [
"D00",
"D01",
"D02",
"D03",
"D04",
"D05",
"D06",
"D07",
"D08",
"D09",
"D10",
"D11"
],
"train_latest_after_holdout_earliest": true
},
"group": {
"train_rows": 673,
"holdout_rows": 221,
"overlapping_depots": [],
"question": "unseen depots within the historical period, not future performance"
},
"temporal_folds": [
{
"train": 170,
"validation": 178,
"train_max_day": 28,
"validation_min_day": 30
},
{
"train": 348,
"validation": 178,
"train_max_day": 57,
"validation_min_day": 60
},
{
"train": 526,
"validation": 178,
"train_max_day": 87,
"validation_min_day": 89
},
{
"train": 704,
"validation": 178,
"train_max_day": 117,
"validation_min_day": 119
}
]
},
"cross_validation": {
"regression": {
"median": {
"mae": [
1.9462022471910112,
1.7711179775280899,
1.8767696629213484,
1.6000056179775284
],
"rmse": [
2.598775797035678,
2.2694012463321536,
2.2831959176370753,
2.123441573949594
]
},
"ridge": {
"mae": [
0.9734455889223046,
0.9522355260291694,
0.8114150433303958,
0.9015101237379638
],
"rmse": [
1.2658335529618507,
1.246991670690505,
1.0625755091655955,
1.1650637838034623
]
},
"forest": {
"mae": [
1.2677668334527978,
1.0301256786542476,
0.9549375662979651,
0.903688980771444
],
"rmse": [
1.6674872416113031,
1.3240881854852151,
1.1766255160929175,
1.1322562907166047
]
}
},
"classification": {
"prior": {
"ap": [
0.2808988764044944,
0.2303370786516854,
0.24719101123595505,
0.2303370786516854
],
"log_loss": [
0.6156695798630484,
0.5400198062709847,
0.559317720224116,
0.5398744054713374
]
},
"logistic": {
"ap": [
0.9457520442126779,
0.7986163947839321,
0.8606151613155248,
0.8245416770747642
],
"log_loss": [
0.199631496991456,
0.3191561073263303,
0.2677123482284254,
0.2695387563476612
]
},
"forest": {
"ap": [
0.8618933326050037,
0.7830614274555758,
0.7388290769055323,
0.8676278913987976
],
"log_loss": [
0.3496118959390845,
0.34196069240174515,
0.35983455203224096,
0.2925595614172929
]
}
}
},
"tuning": [
{
"parameters": {
"model__C": 0.1,
"model__class_weight": null
},
"mean_log_loss": 0.2911228612421971
},
{
"parameters": {
"model__C": 0.1,
"model__class_weight": "balanced"
},
"mean_log_loss": 0.3267269532637338
},
{
"parameters": {
"model__C": 1.0,
"model__class_weight": null
},
"mean_log_loss": 0.26400967722346824
},
{
"parameters": {
"model__C": 1.0,
"model__class_weight": "balanced"
},
"mean_log_loss": 0.30043449340955514
},
{
"parameters": {
"model__C": 10.0,
"model__class_weight": null
},
"mean_log_loss": 0.27482487758239565
},
{
"parameters": {
"model__C": 10.0,
"model__class_weight": "balanced"
},
"mean_log_loss": 0.309351780807611
}
],
"chosen_parameters": {
"model__C": 1.0,
"model__class_weight": null
},
"regression_model": "ridge",
"policy_regression": {
"mae_hours": 0.9408669450496802,
"rmse_hours": 1.3322634142101317,
"r2": 0.694390508968255
},
"policy_uncalibrated_at_05": {
"threshold": 0.5,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8819444444444444,
"balanced_accuracy": 0.8490384615384616,
"precision": 0.7948717948717948,
"recall": 0.775,
"f1": 0.7848101265822784,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.30676112984930476,
"brier": 0.09285306835701318,
"tn": 96,
"fp": 8,
"fn": 9,
"tp": 31,
"alerts": 39,
"alert_fraction": 0.2708333333333333,
"illustrative_cost": 44
},
"policy_calibrated_at_05": {
"threshold": 0.5,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.875,
"balanced_accuracy": 0.8211538461538461,
"precision": 0.8235294117647058,
"recall": 0.7,
"f1": 0.7567567567567568,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 98,
"fp": 6,
"fn": 12,
"tp": 28,
"alerts": 34,
"alert_fraction": 0.2361111111111111,
"illustrative_cost": 54
},
"policy_selected_threshold": {
"threshold": 0.2,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8402777777777778,
"balanced_accuracy": 0.8663461538461539,
"precision": 0.6491228070175439,
"recall": 0.925,
"f1": 0.7628865979381443,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 84,
"fp": 20,
"fn": 3,
"tp": 37,
"alerts": 57,
"alert_fraction": 0.3958333333333333,
"illustrative_cost": 32
},
"threshold_candidates": [
{
"threshold": 0.1,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.7777777777777778,
"balanced_accuracy": 0.8307692307692307,
"precision": 0.5588235294117647,
"recall": 0.95,
"f1": 0.7037037037037037,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 74,
"fp": 30,
"fn": 2,
"tp": 38,
"alerts": 68,
"alert_fraction": 0.4722222222222222,
"illustrative_cost": 38
},
{
"threshold": 0.15,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8194444444444444,
"balanced_accuracy": 0.851923076923077,
"precision": 0.6166666666666667,
"recall": 0.925,
"f1": 0.74,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 81,
"fp": 23,
"fn": 3,
"tp": 37,
"alerts": 60,
"alert_fraction": 0.4166666666666667,
"illustrative_cost": 35
},
{
"threshold": 0.2,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8402777777777778,
"balanced_accuracy": 0.8663461538461539,
"precision": 0.6491228070175439,
"recall": 0.925,
"f1": 0.7628865979381443,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 84,
"fp": 20,
"fn": 3,
"tp": 37,
"alerts": 57,
"alert_fraction": 0.3958333333333333,
"illustrative_cost": 32
},
{
"threshold": 0.25,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8472222222222222,
"balanced_accuracy": 0.8557692307692308,
"precision": 0.6730769230769231,
"recall": 0.875,
"f1": 0.7608695652173914,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 87,
"fp": 17,
"fn": 5,
"tp": 35,
"alerts": 52,
"alert_fraction": 0.3611111111111111,
"illustrative_cost": 37
},
{
"threshold": 0.3,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8541666666666666,
"balanced_accuracy": 0.8605769230769231,
"precision": 0.6862745098039216,
"recall": 0.875,
"f1": 0.7692307692307693,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 88,
"fp": 16,
"fn": 5,
"tp": 35,
"alerts": 51,
"alert_fraction": 0.3541666666666667,
"illustrative_cost": 36
},
{
"threshold": 0.35,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.875,
"balanced_accuracy": 0.8596153846153847,
"precision": 0.75,
"recall": 0.825,
"f1": 0.7857142857142857,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 93,
"fp": 11,
"fn": 7,
"tp": 33,
"alerts": 44,
"alert_fraction": 0.3055555555555556,
"illustrative_cost": 39
},
{
"threshold": 0.4,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8819444444444444,
"balanced_accuracy": 0.8490384615384616,
"precision": 0.7948717948717948,
"recall": 0.775,
"f1": 0.7848101265822784,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 96,
"fp": 8,
"fn": 9,
"tp": 31,
"alerts": 39,
"alert_fraction": 0.2708333333333333,
"illustrative_cost": 44
},
{
"threshold": 0.45,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.875,
"balanced_accuracy": 0.8288461538461538,
"precision": 0.8055555555555556,
"recall": 0.725,
"f1": 0.7631578947368421,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 97,
"fp": 7,
"fn": 11,
"tp": 29,
"alerts": 36,
"alert_fraction": 0.25,
"illustrative_cost": 51
},
{
"threshold": 0.5,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.875,
"balanced_accuracy": 0.8211538461538461,
"precision": 0.8235294117647058,
"recall": 0.7,
"f1": 0.7567567567567568,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 98,
"fp": 6,
"fn": 12,
"tp": 28,
"alerts": 34,
"alert_fraction": 0.2361111111111111,
"illustrative_cost": 54
},
{
"threshold": 0.55,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8611111111111112,
"balanced_accuracy": 0.7961538461538462,
"precision": 0.8125,
"recall": 0.65,
"f1": 0.7222222222222222,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 98,
"fp": 6,
"fn": 14,
"tp": 26,
"alerts": 32,
"alert_fraction": 0.2222222222222222,
"illustrative_cost": 62
},
{
"threshold": 0.6,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8541666666666666,
"balanced_accuracy": 0.7759615384615384,
"precision": 0.8275862068965517,
"recall": 0.6,
"f1": 0.6956521739130435,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 99,
"fp": 5,
"fn": 16,
"tp": 24,
"alerts": 29,
"alert_fraction": 0.2013888888888889,
"illustrative_cost": 69
},
{
"threshold": 0.65,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8402777777777778,
"balanced_accuracy": 0.7432692307692308,
"precision": 0.84,
"recall": 0.525,
"f1": 0.6461538461538462,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 100,
"fp": 4,
"fn": 19,
"tp": 21,
"alerts": 25,
"alert_fraction": 0.1736111111111111,
"illustrative_cost": 80
},
{
"threshold": 0.7,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8402777777777778,
"balanced_accuracy": 0.7432692307692308,
"precision": 0.84,
"recall": 0.525,
"f1": 0.6461538461538462,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 100,
"fp": 4,
"fn": 19,
"tp": 21,
"alerts": 25,
"alert_fraction": 0.1736111111111111,
"illustrative_cost": 80
},
{
"threshold": 0.75,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.8125,
"balanced_accuracy": 0.6855769230769231,
"precision": 0.8421052631578947,
"recall": 0.4,
"f1": 0.5423728813559322,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 101,
"fp": 3,
"fn": 24,
"tp": 16,
"alerts": 19,
"alert_fraction": 0.13194444444444445,
"illustrative_cost": 99
},
{
"threshold": 0.8,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.7847222222222222,
"balanced_accuracy": 0.6278846153846154,
"precision": 0.8461538461538461,
"recall": 0.275,
"f1": 0.41509433962264153,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 102,
"fp": 2,
"fn": 29,
"tp": 11,
"alerts": 13,
"alert_fraction": 0.09027777777777778,
"illustrative_cost": 118
},
{
"threshold": 0.85,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.7777777777777778,
"balanced_accuracy": 0.6153846153846154,
"precision": 0.8333333333333334,
"recall": 0.25,
"f1": 0.38461538461538464,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 102,
"fp": 2,
"fn": 30,
"tp": 10,
"alerts": 12,
"alert_fraction": 0.08333333333333333,
"illustrative_cost": 122
},
{
"threshold": 0.9,
"n": 144,
"prevalence": 0.2777777777777778,
"accuracy": 0.7569444444444444,
"balanced_accuracy": 0.5701923076923077,
"precision": 0.8571428571428571,
"recall": 0.15,
"f1": 0.2553191489361702,
"roc_auc": 0.9319711538461538,
"average_precision": 0.7846350470387247,
"log_loss": 0.3017381476843004,
"brier": 0.09465169015492735,
"tn": 103,
"fp": 1,
"fn": 34,
"tp": 6,
"alerts": 7,
"alert_fraction": 0.04861111111111111,
"illustrative_cost": 137
}
],
"policy_regression_by_zone": [
{
"zone": "central",
"n": 60,
"mae_hours": 0.9768835236495235,
"rmse_hours": 1.418540950620956,
"r2": 0.6854279945171019
},
{
"zone": "east",
"n": 50,
"mae_hours": 0.9518567585815595,
"rmse_hours": 1.2457212963426312,
"r2": 0.6285833528240199
},
{
"zone": "west",
"n": 34,
"mae_hours": 0.8611467864442514,
"rmse_hours": 1.2977665517202766,
"r2": 0.7285160578770014
}
],
"permutation_importance": {
"distance_km": {
"mean_log_loss_increase": 0.23837613745416272,
"repeat_sd": 0.07018340359641186
},
"queued_jobs": {
"mean_log_loss_increase": 0.17424279995827083,
"repeat_sd": 0.026859697107030286
},
"item_count": {
"mean_log_loss_increase": 0.03914361378864149,
"repeat_sd": 0.00805597280038388
},
"weekday": {
"mean_log_loss_increase": 0.0006080281400483112,
"repeat_sd": 0.00020570404572789743
},
"promised_hours": {
"mean_log_loss_increase": 0.23080256009634742,
"repeat_sd": 0.040337223455098214
},
"priority": {
"mean_log_loss_increase": 0.01721767992669492,
"repeat_sd": 0.015289167990484152
},
"zone": {
"mean_log_loss_increase": 0.032062970017706804,
"repeat_sd": 0.015674343045305076
},
"depot_id": {
"mean_log_loss_increase": 0.004328174515102101,
"repeat_sd": 0.01425701121528372
}
},
"versions": {
"python": "3.12.14",
"numpy": "2.3.5",
"pandas": "2.2.3",
"scipy": "1.17.0",
"scikit-learn": "1.8.0",
"joblib": "1.5.3",
"threadpoolctl": "3.6.0"
}
}
