"""Public learner checks; does NOT grade a written report or certify leakage safety."""
from pathlib import Path
import argparse
import importlib.util
import numpy as np
from sklearn.base import clone
from sklearn.pipeline import Pipeline
from courseml.data import FEATURES,FORBIDDEN,load_part
from courseml.modeling import choose_threshold
def load(path):
spec=importlib.util.spec_from_file_location('learner_submission',path)
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
return module
def check_functions(module):
assert np.isclose(module.regression_mae([2,4,8],[3,3,6]),4/3)
assert module.confusion_counts([0,0,1,1],[.1,.6,.4,.8],.5)=={'tn':1,'fp':1,'fn':1,'tp':1}
assert module.threshold_cost([0,0,1,1],[.1,.6,.4,.8],.5)==5
assert module.candidate_features(FEATURES+FORBIDDEN+['unapproved'])==FEATURES
print('4 public learner function checks passed.')
def check_capstone(module):
train=load_part('train');model=module.build_classifier()
assert isinstance(model,Pipeline),'Return a full preprocessing + estimator Pipeline.'
model=clone(model);model.fit(train[FEATURES],train.late)
rows=train[FEATURES].head(5).copy();rows.loc[rows.index[0],'depot_id']='D_NEW'
probability=model.predict_proba(rows)[:,1]
assert len(probability)==5 and np.isfinite(probability).all()
assert ((probability>=0)&(probability<=1)).all()
for y,p in [([0,0,1,1,1],[.1,.2,.3,.7,.9]),([0,1],[1.,1.]),([0,1],[.5,.5])]:
assert np.isclose(module.choose_policy(np.array(y),np.array(p)),choose_threshold(y,p)[0])
regressor=module.build_regressor()
assert isinstance(regressor,Pipeline),'Return a preprocessing + regressor Pipeline.'
regressor=clone(regressor).fit(train[FEATURES],train.duration_hours)
values=regressor.predict(rows)
assert len(values)==5 and np.isfinite(values).all()
print('Classification/regression pipeline smoke and 3 threshold-policy checks passed. Human review still required.')
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--capstone',action='store_true')
parser.add_argument('--file',type=Path);args=parser.parse_args()
default='exercises/capstone.py' if args.capstone else 'exercises/learner_functions.py'
try:
module=load(args.file or Path(__file__).resolve().parent/default)
(check_capstone if args.capstone else check_functions)(module)
except NotImplementedError as exc:
parser.exit(1,f'Your starter is intentionally incomplete: {exc}\n')
