The script is inent to build a baseline model for creditcard fraud detection.¶
In [23]:
import os,datetime
import pandas as pd
import numpy as np
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import roc_curve, auc, average_precision_score,f1_score,fbeta_score,precision_recall_curve
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
from sklearn.model_selection import cross_val_score,cross_validate,train_test_split,StratifiedKFold
Load data and engineer features as discussed in explanatory analysis¶
In [13]:
creditcard=pd.read_csv('creditcard.csv')
In [14]:
df2=creditcard
df2['Hour']=df2['Time'].apply(lambda x : datetime.datetime.fromtimestamp(x).hour)
df2['Day']=df2['Time'].apply(lambda x : 0 if datetime.datetime.fromtimestamp(x).day==31 else 1)
df2['Amount_A']=np.log10(creditcard['Amount']+1)
df2['V1_A']=np.log10(-creditcard['V1']+3)
Create one-hot vectors for Hour.¶
In [15]:
Hour= pd.get_dummies(df2['Hour'],prefix='Hour')
df2 = pd.concat([df2,Hour],axis=1)
df2 = df2.drop(["Hour"], axis=1)
remove=["Time","V1","V6","V8","V13","V15","V19","V20","V21","V22","V23","V24","V25","V26","V27","V28","Amount","Amount_A"]
df2= df2.drop(remove, axis =1)
In [16]:
df2.describe
Out[16]:
In [17]:
X=df2.iloc[:,df2.columns != 'Class'].as_matrix()
y=df2['Class'].as_matrix()
Split dataset into training set and test set.¶
In [18]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42,stratify=y)
Define a neural network classifier.¶
In [19]:
layers=(100,100,100,100)
learning_rate_init=0.001
alpha=0.0001
#clf = MLPClassifier(solver='adam',activation='tanh',hidden_layer_sizes=layers, learning_rate='adaptive',learning_rate_init=lr_init,alpha=alpha,random_state=2)
clf = MLPClassifier(solver='adam',activation='tanh',
hidden_layer_sizes=layers, learning_rate='adaptive',
learning_rate_init=learning_rate_init,alpha=alpha,random_state=2345,
max_iter=200,verbose=10)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
Scoring the model using 5-fold cross_validation.¶
In [20]:
scorings ='average_precision'
fold=5
cv=StratifiedKFold(n_splits=fold,random_state=1234)
scores=cross_validate(clf,X_train_scaled,y_train,cv=cv,scoring=scorings,return_train_score=True)
print(scores)
In [21]:
np.mean(scores['test_score'])
Out[21]:
Precision-recall Curve for 5-fold cross validation.¶
In [24]:
def plot_precision_recall(X,y,cv):
aps=[]
f1scores=[]
f2scores=[]
best_thresholds_f1=[]
best_thresholds_f2=[]
i = 0
for train, test in cv.split(X, y):
scaler = StandardScaler()
scaler.fit(X[train])
X[train] = scaler.transform(X[train])
X[test] = scaler.transform(X[test])
probas_ = clf.fit(X[train], y[train]).predict_proba(X[test])[:, 1]
# Compute precision recall curve and area the curve
precision, recall, thresholds = precision_recall_curve(y[test], probas_)
average_precision = average_precision_score(y[test], probas_)
# auc = metrics.auc(recall, precision)
plt.step(recall, precision,lw=1, alpha=0.3,
label=r'f%d AP=%0.2f' % (i,average_precision))
aps.append(average_precision)
f1score=2*precision*recall/(precision+recall)
f2score=5*precision*recall/(4*precision+recall)
f1scores.append(np.amax(f1score))
f2scores.append(np.amax(f2score))
best_thresholds_f1.append(thresholds[np.argmax(f1score)])
best_thresholds_f2.append(thresholds[np.argmax(f2score)])
# print("Fold=%d max F1 score" % i)
# print(metrics.classification_report(y[test],proba_temp))
# print(metrics.confusion_matrix(y[test],proba_temp))
# print("Fold=%d max F2 score" % i)
# print(metrics.classification_report(y[test],proba_temp))
# print(metrics.confusion_matrix(y[test],proba_temp))
i += 1
plt.plot([0, 3], [0, 3], linestyle='--', lw=2, color='r',
label='Luck', alpha=.8)
mean_aps = np.mean(aps, axis=0)
std_aps= np.std(aps)
#mean_tpr[-1] = 1.0
#mean_auc = auc(mean_fpr, mean_tpr)
#plt.plot(mean_fpr, mean_tpr, color='b',
# label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
# lw=2, alpha=.8)
#std_tpr = np.std(tprs, axis=0)
#tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
#tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
#plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
# label=r'$\pm$ 1 std. dev.')
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall-Curve mean %0.2f $\pm$ %0.2f)' % (mean_aps, std_aps))
plt.legend(loc="lower left")
plt.show()
return f1scores,f2scores,best_thresholds_f1,best_thresholds_f2
f1scores,f2scores,best_thresholds_f1,best_thresholds_f2=plot_precision_recall(X_train,y_train,cv)
The plot shows the model can achieve around 0.8 recall as well as keep the precision around 0.8. That measns the model can detect around 80% of frauds without interfering a lot of customers. However, the curves drop down rapidly when recall is large than 0.8. So if you want to increase the recall above 80%, the pay-off is very expensive as a great number of customers being inspected but only gains a few more frauds being found.
Predict the test set.¶
In [25]:
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
probas_ = clf.fit(X_train_scaled, y_train).predict_proba(X_test_scaled)[:, 1]
y_score=clf.predict_proba(X_test_scaled)[:,1]
#precision, recall, thresholds = precision_recall_curve(y_test, probas_)
average_precision = average_precision_score(y_test, probas_)
predict=(probas_>0.5).astype(int)
f1score=f1_score(y_test,predict)
f2score=fbeta_score(y_test,predict,beta=2)
print("Ap= ",average_precision)
print("f1 score: ",f1score)
print("f2 score: ",f2score)
print(metrics.classification_report(y_test,predict))
print(metrics.confusion_matrix(y_test,predict))