本文整理汇总了Python中ml_metrics.quadratic_weighted_kappa函数的典型用法代码示例。如果您正苦于以下问题:Python quadratic_weighted_kappa函数的具体用法?Python quadratic_weighted_kappa怎么用?Python quadratic_weighted_kappa使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了quadratic_weighted_kappa函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_quadratic_weighted_kappa
def test_quadratic_weighted_kappa(self):
kappa = metrics.quadratic_weighted_kappa([1,2,3],[1,2,3])
self.assertAlmostEqual(kappa, 1.0)
kappa = metrics.quadratic_weighted_kappa([1,2,1],[1,2,2],1,2)
self.assertAlmostEqual(kappa, 0.4)
kappa = metrics.quadratic_weighted_kappa([1,2,3,1,2,2,3],[1,2,3,1,2,3,2])
self.assertAlmostEqual(kappa, 0.75)
示例2: predict
def predict(self, model, xg_train, xg_test, objective='reg:linear'):
"""
Parameters
----------
model : xgboost.Booster
xgboost model ready for making predictions
xg_train : xgboost.DMatrix
training data
xg_test : xgboost.DMatrix
testing data
Returns
-------
model_prediction : ModelPrediction (named tuple)
"""
train_score = model.predict(
xg_train, ntree_limit=model.best_iteration)
test_score = model.predict(
xg_test, ntree_limit=model.best_iteration)
train_label = np.asarray(xg_train.get_label())
test_label = np.asarray(xg_test.get_label())
if objective == 'reg:linear':
# Cuttofs are optimized here
best_cuts = optimize_cutoffs(train_score, train_label,
verbose=False)
train_prediction = classify_with_cutoffs(train_score, best_cuts)
test_prediction = classify_with_cutoffs(test_score, best_cuts)
else:
train_prediction = train_score
test_prediction = test_score
train_qwk = quadratic_weighted_kappa(train_label, train_prediction)
test_qwk = quadratic_weighted_kappa(test_label, test_prediction)
return ModelPrediction(train_label, test_label,
train_score, test_score,
train_prediction, test_prediction,
train_qwk, test_qwk,
precision_score(train_label, train_prediction,
average=None),
precision_score(test_label, test_prediction,
average=None)
)
示例3: minimize_quadratic_weighted_kappa
def minimize_quadratic_weighted_kappa(cutpoints,y_pred=None,y=None):
cutpoints = np.sort(cutpoints)
cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
y_pred = pd.cut(y_pred,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
score = quadratic_weighted_kappa(y,y_pred)
print score
return -score
示例4: predict_score
def predict_score():
file = open("model/predictions.txt")
numarray = []
while 1:
line = file.readline()
if not line:
break
numarray.append(int(float(line)))
file = open("model/answers.txt")
answerarray = []
while 1:
line = file.readline()
if not line:
break
answerarray.append(int(float(line)))
##print len(numarray)
##print len(answerarray)
solutionarray = []
for x in range(0, len(numarray)):
if numarray[x] == answerarray[x]:
solutionarray.append(1)
else:
solutionarray.append(0)
onecounter = solutionarray.count(1)
print "QWK_Score: " + str(metrics.quadratic_weighted_kappa(answerarray,numarray))
示例5: eval_dag
def eval_dag(dag, filename, dag_id=None):
dag = normalize_dag(dag)
if filename not in input_cache:
input_cache[filename] = pd.read_csv('data/'+filename, sep=';')
data = input_cache[filename]
feats = data[data.columns[:-1]]
targets = data[data.columns[-1]]
le = preprocessing.LabelEncoder()
ix = targets.index
targets = pd.Series(le.fit_transform(targets), index=ix)
errors = []
start_time = time.time()
for train_idx, test_idx in cross_validation.StratifiedKFold(targets, n_folds=5):
train_data = (feats.iloc[train_idx], targets.iloc[train_idx])
test_data = (feats.iloc[test_idx], targets.iloc[test_idx])
ms = train_dag(dag, train_data)
preds = test_dag(dag, ms, test_data)
acc = mm.quadratic_weighted_kappa(test_data[1], preds)
errors.append(acc)
m_errors = float(np.mean(errors))
s_errors = float(np.std(errors))
return m_errors, s_errors, time.time() - start_time
示例6: _score_offset
def _score_offset(self, bin_offset, sv):
flg = self._data[:, 0].astype(int) == sv
self._data[flg, 1] = self._data[flg, 0] + bin_offset
offset_pred = np.clip(np.round(self._data[:, 1]), 1, 8)\
.astype(int)
kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)
return -kappa
示例7: evalerror_softmax_cdf
def evalerror_softmax_cdf(preds, dtrain, cdf):
## label are in [0,1,2,3]
labels = dtrain.get_label() + 1
preds = getClfScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
示例8: keras_model
def keras_model():
import pandas as pd
import numpy as np
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
from data_util import load_csvs, load_other
import ml_metrics as metrics
nb_words = 6500
maxlen = 175
filter_length = 10
other_col_dim = 4
X_train, Y_train, X_test, Y_test, nb_classes = load_csvs('data/tpov4/train_1.csv',
'data/tpov4/test_1.csv',
nb_words, maxlen, 'self', w2v=None)
# read _other.csv
other_train = load_other('data/tpov4/train_1_other.csv', maxlen, other_col_dim)
other_test = load_other('data/tpov4/test_1_other.csv', maxlen, other_col_dim)
print('other tensor:', other_train.shape)
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Convolution1D(nb_filter=50,
filter_length=filter_length,
border_mode="valid", activation="relu",
input_shape=(maxlen, other_col_dim)))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.05))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer={{choice(['rmsprop', 'adam', 'adadelta', 'adagrad'])}})
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(other_train, Y_train, batch_size=32, nb_epoch=25,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(other_test, batch_size=32)
org_classes = np_utils.categorical_probas_to_classes(Y_test)
acc = np_utils.accuracy(classes, org_classes) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, org_classes)
print('Test Kappa:', kappa)
return {'loss': -acc, 'status': STATUS_OK}
示例9: eval_wrapper
def eval_wrapper(yhat, y):
"""
Evaluation metric for the competition : quad weighted kappa
"""
y = np.array(y)
y = y.astype(int)
yhat = np.array(yhat)
yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)
return quadratic_weighted_kappa(yhat, y)
示例10: xgb_regression_quadratic_weighted_kappa
def xgb_regression_quadratic_weighted_kappa(preds,dtrain):
labels = dtrain.get_label()
cutpoints = [1.886638,3.303624,4.152756,4.825063,5.653934,6.236325,6.765184]
res = minimize(minimize_quadratic_weighted_kappa,cutpoints,(preds,labels),method='BFGS')
cutpoints = np.sort(res.x)
cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
y_pred = pd.cut(preds,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
kappa = quadratic_weighted_kappa(labels,y_pred)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
示例11: evalerror_cocr_cdf
def evalerror_cocr_cdf(preds, dtrain, cdf):
labels = dtrain.get_label() + 1
#print preds.shape
## get prediction
#preds = sigmoid(preds)
preds = applyCOCRRule(preds)
preds = getScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
示例12: evalerror
def evalerror(preds, dtrain):
## label are in [0,1,2,3] as required by XGBoost for multi-classification
labels = dtrain.get_label() + 1
## class probability
preds = softmax(preds)
## decoding (naive argmax decoding)
pred_labels = np.argmax(preds, axis=1) + 1
## compute quadratic weighted kappa (using implementation from @Ben Hamner
## https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/quadratic_weighted_kappa.py
kappa = quadratic_weighted_kappa(labels, pred_labels)
return 'kappa', kappa
示例13: _offset_qwk_score
def _offset_qwk_score(self, offset):
"""
:param numpy.array offset:
:param numpy.array y_true:
:param numpy.array y_pred:
:rtype: float
"""
offset_pred = self._apply_offset(self._data, offset)
kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)
return -kappa
示例14: cnn1d_selfembd
def cnn1d_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_dim,
nb_filter, filter_length, batch_size, nb_epoch, optm):
"""
- CNN-1d on text input (represented in int)
- MOT
- dropout + L2 softmax
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param vocab_size
:param embd_dim
:param nb_filter
:param filter_length
:param batch_size
:param nb_epoch
:param optm optimizer options, e.g., adam, rmsprop, etc.
:return:
"""
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu"))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test accuracy:', acc)
# return(acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
示例15: on_epoch_end
def on_epoch_end(self, epoch, logs={}):
p = self.model.predict(self.X_val.values, verbose=0)
current = ml_metrics.quadratic_weighted_kappa(self.y_val.values.ravel(),np.clip(np.round(p.astype(int).ravel()), 1, 8))
if current > self.best:
self.best = current
self.wait = 0
else:
if self.wait >= self.patience:
self.model.stop_training = True
print('Epoch %05d: early stopping' % (epoch))
self.wait += 1 #incremental the number of times without improvement
print('Epoch %d Kappa: %f | Best Kappa: %f \n' % (epoch,current,self.best))