Python model_selection.cross_validate方法代码示例

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def eval_by_cv(self, X, Y, n_splits=5, verbose=True):
    """ Fits the conditional density model with cross-validation by using the score function of the BaseDensityEstimator for
    scoring the various splits.

    Args:
      X: numpy array to be conditioned on - shape: (n_samples, n_dim_x)
      Y: numpy array of y targets - shape: (n_samples, n_dim_y)
      n_splits: number of cross-validation folds (positive integer)
      verbose: the verbosity level
    """
    X, Y = self._handle_input_dimensionality(X, Y, fitting=True)
    cv_results = cross_validate(self, X=X, y=Y, cv=n_splits, return_estimator=True, verbose=verbose)

    test_scores = cv_results['test_score']
    test_scores_max_idx = np.nanargmax(test_scores)
    estimator = cv_results['estimator'][test_scores_max_idx]

    self.set_params(**estimator.get_params())
    self.fit(X, Y) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def score_models(clf, X, y, encoder, runs=1):
    """
    Takes in a classifier that supports multiclass classification, and X and a y, and returns a cross validation score.

    """

    scores = []

    X_test = None
    for _ in range(runs):
        X_test = encoder().fit_transform(X, y)

        # Some models, like logistic regression, like normalized features otherwise they underperform and/or take a long time to converge.
        # To be rigorous, we should have trained the normalization on each fold individually via pipelines.
        # See grid_search_example to learn how to do it.
        X_test = StandardScaler().fit_transform(X_test)

        scores.append(cross_validate(clf, X_test, y, n_jobs=1, cv=5)['test_score'])
        gc.collect()

    scores = [y for z in [x for x in scores] for y in z]

    return float(np.mean(scores)), float(np.std(scores)), scores, X_test.shape[1] 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def test_diff_detector_cross_validate(return_estimator: bool):
    """
    DiffBasedAnomalyDetector.cross_validate implementation should be the
    same as sklearn.model_selection.cross_validate if called the same.

    And it always will update `return_estimator` to True, as it requires
    the intermediate models to calculate the thresholds
    """
    X = np.random.random((100, 10))
    y = np.random.random((100, 1))

    model = DiffBasedAnomalyDetector(base_estimator=LinearRegression())

    cv = TimeSeriesSplit(n_splits=3)
    cv_results_da = model.cross_validate(
        X=X, y=y, cv=cv, return_estimator=return_estimator
    )
    cv_results_sk = cross_validate(model, X=X, y=y, cv=cv, return_estimator=True)

    assert cv_results_da.keys() == cv_results_sk.keys() 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def test_diff_detector_require_thresholds(require_threshold: bool):
    """
    Should fail if requiring thresholds, but not calling cross_validate
    """
    X = pd.DataFrame(np.random.random((100, 5)))
    y = pd.DataFrame(np.random.random((100, 2)))

    model = DiffBasedAnomalyDetector(
        base_estimator=MultiOutputRegressor(LinearRegression()),
        require_thresholds=require_threshold,
    )

    model.fit(X, y)

    if require_threshold:
        # FAIL: Forgot to call .cross_validate to calculate thresholds.
        with pytest.raises(AttributeError):
            model.anomaly(X, y)

        model.cross_validate(X=X, y=y)
        model.anomaly(X, y)
    else:
        # thresholds not required
        model.anomaly(X, y) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def mae_cv(self, cv):
        """
        This method performs cross-validation over median absolute error.
        
        Parameters
        ----------
        * cv : integer
          The number of cross validation folds to perform

        Returns
        -------
        Returns a scores of the k-fold median absolute error.
        """

        mae = metrics.make_scorer(metrics.median_absolute_error)
        result = cross_validate(self.reg, self.X,
                                self.y, cv=cv,
                                scoring=(mae))
        return self.get_test_score(result) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def mse_cv(self, cv):
        """
        This method performs cross-validation over mean squared error.
        
        Parameters
        ----------
        * cv : integer
          The number of cross validation folds to perform

        Returns
        -------
        Returns a scores of the k-fold mean squared error.
        """
        mse = metrics.make_scorer(metrics.mean_squared_error)
        result = cross_validate(self.reg, self.X,
                                self.y, cv=cv,
                                scoring=(mse))
        return self.get_test_score(result) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def tse_cv(self, cv):
        """
        This method performs cross-validation over trimean squared error.
        
        Parameters
        ----------
        * cv : integer
          The number of cross validation folds to perform

        Returns
        -------
        Returns a scores of the k-fold trimean squared error.
        """
        tse = metrics.make_scorer(self.trimean_squared_error)
        result = cross_validate(self.reg, self.X,
                                self.y, cv=cv,
                                scoring=(tse))
        return self.get_test_score(result) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def tae_cv(self, cv):
        """
        This method performs cross-validation over trimean absolute error.
        
        Parameters
        ----------
        * cv : integer
          The number of cross validation folds to perform

        Returns
        -------
        Returns a scores of the k-fold trimean absolute error.
        """
        tse = metrics.make_scorer(self.trimean_absolute_error)
        result = cross_validate(self.reg, self.X,
                                self.y, cv=cv,
                                scoring=(tse))
        return self.get_test_score(result) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def test_cross_validate_return_train_score_warn():
    # Test that warnings are raised. Will be removed in 0.21

    X, y = make_classification(random_state=0)
    estimator = MockClassifier()

    result = {}
    for val in [False, True, 'warn']:
        result[val] = assert_no_warnings(cross_validate, estimator, X, y,
                                         return_train_score=val)

    msg = (
        'You are accessing a training score ({!r}), '
        'which will not be available by default '
        'any more in 0.21. If you need training scores, '
        'please set return_train_score=True').format('train_score')
    train_score = assert_warns_message(FutureWarning, msg,
                                       result['warn'].get, 'train_score')
    assert np.allclose(train_score, result[True]['train_score'])
    assert 'train_score' not in result[False] 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def __init__(
        self,
        base_estimator: BaseEstimator = KerasAutoEncoder(kind="feedforward_hourglass"),
        scaler: TransformerMixin = RobustScaler(),
        require_thresholds: bool = True,
        window=None,
    ):
        """
        Classifier which wraps a ``base_estimator`` and provides a diff error
        based approach to anomaly detection.

        It trains a ``scaler`` to the target **after** training, purely for
        error calculations. The underlying ``base_estimator`` is trained
        with the original, unscaled, ``y``.

        Parameters
        ----------
        base_estimator: sklearn.base.BaseEstimator
            The model to which normal ``.fit``, ``.predict`` methods will be used.
            defaults to py:class:`gordo.machine.model.models.KerasAutoEncoder` with
            ``kind='feedforward_hourglass``
        scaler: sklearn.base.TransformerMixin
            Defaults to ``sklearn.preprocessing.RobustScaler``
            Used for transforming model output and the original ``y`` to calculate
            the difference/error in model output vs expected.
        require_thresholds: bool
            Requires calculating ``thresholds_`` via a call to :func:`~DiffBasedAnomalyDetector.cross_validate`.
            If this is set (default True), but :func:`~DiffBasedAnomalyDetector.cross_validate`
            was not called before calling :func:`~DiffBasedAnomalyDetector.anomaly` an ``AttributeError``
            will be raised.
        window: int
            Window size for smoothed thresholds
        """
        self.base_estimator = base_estimator
        self.scaler = scaler
        self.require_thresholds = require_thresholds
        self.window = window 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def test_cross_validate_many_jobs():
    # regression test for #12154: cv='warn' with n_jobs>1 trigger a copy of
    # the parameters leading to a failure in check_cv due to cv is 'warn'
    # instead of cv == 'warn'.
    X, y = load_iris(return_X_y=True)
    clf = SVC(gamma='auto')
    grid = GridSearchCV(clf, param_grid={'C': [1, 10]})
    cross_validate(grid, X, y, n_jobs=2) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def model_evaluate(X, Y, pipeline):
    try:
        results = []
        if "scoring" in pipeline["options"]:
            if len(pipeline['options']['scoring']) > 0:
                scoring = pipeline['options']['scoring']
            else:
                scoring = "neg_mean_squared_error"
        else:
            scoring = "neg_mean_squared_error"

        kfold = 10
        if "kfold" in pipeline['options']:
            kfold = int(pipeline["options"]["kfold"])

        model = scikitlearn.getSKLearnModel(pipeline['options']['model_name'])
        valresult = cross_validate(model, X, Y, cv=kfold, scoring=scoring, return_train_score=True)

        model.fit(X, Y)
        for p in valresult:
            results.append({"param": p, "values": valresult[p].tolist(), "min": valresult[p].min, "max": valresult[p].max});
        output = jsonpickle.encode(results, unpicklable=False)
        projectmgr.UpdateExecuteResult(jobid, output)
        picklefile = projectfolder + "/model.out"
        with open(picklefile, "wb") as f:
            pickle.dump(model, f)

        return output
    except Exception as e:
        raise Exception("model_evaluate: " + str(e)) 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def _get_cv_score(self, feature_to_remove):
        X, fit_params = self.dataset.getX(feature_to_remove=feature_to_remove, fit_params=self.fit_params)
        y = self.dataset.y

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            cv_results = cross_validate(self.model, X, y, cv=self.cv, scoring=self.scoring, fit_params=fit_params)
        return cv_results['test_score'] 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def __call__(self, trial):
        # type: (trial_module.Trial) -> float

        estimator = clone(self.estimator)
        params = self._get_params(trial)

        estimator.set_params(**params)

        if self.enable_pruning:
            scores = self._cross_validate_with_pruning(trial, estimator)
        else:
            scores = cross_validate(
                estimator,
                self.X,
                self.y,
                cv=self.cv,
                error_score=self.error_score,
                fit_params=self.fit_params,
                groups=self.groups,
                return_train_score=self.return_train_score,
                scoring=self.scoring,
            )

        self._store_scores(trial, scores)

        return trial.user_attrs["mean_test_score"] 

# 需要导入模块: from sklearn import model_selection [as 别名]
# 或者: from sklearn.model_selection import cross_validate [as 别名]
def get_cv_scores(estimator, X, y, scoring, cv=5):
    return cross_validate(estimator, X, y, cv=cv, n_jobs=1,
                          scoring=scoring,
                          return_train_score=False)