Learn with Random-Forest and compare Cross-Validation methods

This example shows how to make a classification with different cross-validation methods.

Import librairies

from museotoolbox.ai import SuperLearner
from museotoolbox import cross_validation
from museotoolbox.processing import extract_ROI
from museotoolbox import datasets
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold

Load HistoricalMap dataset

raster,vector = datasets.load_historical_data(low_res=True)
field = 'Class'
group = 'uniquefid'
X,y,g = extract_ROI(raster,vector,field,group)

Initialize Random-Forest

classifier = RandomForestClassifier(random_state=12,n_jobs=1)

Create list of different CV

CVs = [cross_validation.RandomStratifiedKFold(n_splits=2),
       cross_validation.LeavePSubGroupOut(valid_size=0.5),
       cross_validation.LeaveOneSubGroupOut(),
       StratifiedKFold(n_splits=2,shuffle=True) #from sklearn
       ]

kappas=[]



for cv in CVs :
    SL = SuperLearner( classifier=classifier,param_grid=dict(n_estimators=[50,100]),n_jobs=1)
    SL.fit(X,y,group=g,cv=cv)
    print('Kappa for '+str(type(cv).__name__))
    cvKappa = []

    for stats in SL.get_stats_from_cv(confusion_matrix=False,kappa=True):
        print(stats['kappa'])
        cvKappa.append(stats['kappa'])

    kappas.append(cvKappa)

    print(20*'=')

Out:

Kappa for RandomStratifiedKFold
0.9188656472654342
0.9223905038224895
====================
Kappa for LeavePSubGroupOut
0.8179639997591958
0.7826805310852983
====================
Kappa for LeaveOneSubGroupOut
0.8207759170686313
0.845428586696654
====================
Kappa for StratifiedKFold
0.907135589057438
0.9247822738313881
====================

Plot example

from matplotlib import pyplot as plt
plt.title('Kappa according to Cross-validation methods')
plt.boxplot(kappas,labels=[str(type(i).__name__) for i in CVs], patch_artist=True)
plt.grid()
plt.ylabel('Kappa')
plt.xticks(rotation=15)
plt.show()