Python h2o.H2OFrame方法代码示例

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def fit_transform(self, X: dt.Frame, y: np.array = None):
        h2o.init(port=config.h2o_recipes_port)
        model = H2OAutoEncoderEstimator(activation='tanh', epochs=1, hidden=[50, 50], reproducible=True, seed=1234)
        frame = h2o.H2OFrame(X.to_pandas())
        model_path = None
        try:
            model.train(x=list(range(X.ncols)), training_frame=frame)
            self.id = model.model_id
            model_path = os.path.join(user_dir(), "h2o_model." + str(uuid.uuid4()))
            model_path = h2o.save_model(model=model, path=model_path)
            with open(model_path, "rb") as f:
                self.raw_model_bytes = f.read()
            return model.anomaly(frame).as_data_frame(header=False)
        finally:
            if model_path is not None:
                remove(model_path)
            h2o.remove(model) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def transform(self, X: dt.Frame):
        h2o.init(port=config.h2o_recipes_port)
        model_path = os.path.join(user_dir(), self.id)
        model_file = os.path.join(model_path, "h2o_model." + str(uuid.uuid4()) + ".bin")
        os.makedirs(model_path, exist_ok=True)
        with open(model_file, "wb") as f:
            f.write(self.raw_model_bytes)
        model = h2o.load_model(os.path.abspath(model_file))
        frame = h2o.H2OFrame(X.to_pandas())
        anomaly_frame = None

        try:
            anomaly_frame = model.anomaly(frame)
            anomaly_frame_df = anomaly_frame.as_data_frame(header=False)
            return anomaly_frame_df
        finally:
            remove(model_file)
            h2o.remove(self.id)
            h2o.remove(anomaly_frame) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def predict(self, X, **kwargs):
        model, _, _, _ = self.get_model_properties()
        X = dt.Frame(X)
        h2o.init(port=config.h2o_recipes_port, log_dir=self.my_log_dir)
        model_path = os.path.join(user_dir(), self.id)
        model_file = os.path.join(model_path, "h2o_model." + str(uuid.uuid4()) + ".bin")
        os.makedirs(model_path, exist_ok=True)
        with open(model_file, "wb") as f:
            f.write(model)
        model = h2o.load_model(os.path.abspath(model_file))
        test_frame = h2o.H2OFrame(X.to_pandas(), column_types=self.col_types)
        preds_frame = None

        try:
            preds_frame = model.predict(test_frame)
            preds = preds_frame.as_data_frame(header=False)

            return preds.values.ravel()

        finally:
            remove(model_file)
            # h2o.remove(self.id) # Cannot remove id, do multiple predictions on same model
            h2o.remove(test_frame)
            if preds_frame is not None:
                h2o.remove(preds_frame) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def h2o_compute(X_train, y_train, X_test):
    import h2o
    from h2o.automl import H2OAutoML
    h2o.init(ip='localhost', port='55555', min_mem_size='14g', max_mem_size='15g')
    aml = H2OAutoML(max_runtime_secs = 3600)
    dd = h2o.H2OFrame(pd.concat([X_train, y_train], axis=1))
    dd['target'] = dd['target'].asfactor()
    aml.train(y = 'target', training_frame = dd)
    response = aml.predict(h2o.H2OFrame(X_test))
    return np.array(response[:,2].as_data_frame()) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def transform(self, X: dt.Frame):

        stop_column_name = self.__class__._stop_column_name
        if stop_column_name in X.names:
            del X[:, stop_column_name]
        else:
            return X

        if self.id is None:
            return X

        #self._output_feature_names = list(X.names)
        #self._feature_desc = list(X.names)

        h2o.init(port=config.h2o_recipes_port, log_dir=self.my_log_dir)
        model_path = os.path.join(temporary_files_path, self.id)
        with open(model_path, "wb") as f:
            f.write(self.raw_model_bytes)
        model = h2o.load_model(os.path.abspath(model_path))
        remove(model_path)

        frame = h2o.H2OFrame(X.to_pandas())
        try:
            risk_frame = model.predict(frame)
            X[:, "risk_score_coxph_{}_{}".format(self.ties, self.max_iterations)] = risk_frame.as_data_frame(header=False)
            return X
        finally:
            h2o.remove(self.id)
            h2o.remove(frame)
            if risk_frame is not None:
                h2o.remove(risk_frame) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def predict(self, X, **kwargs):
        model, _, _, _ = self.get_model_properties()
        X = dt.Frame(X)
        h2o.init(port=config.h2o_recipes_port, log_dir=self.my_log_dir)
        model_path = os.path.join(user_dir(), self.id)
        model_file = os.path.join(model_path, "h2o_model." + str(uuid.uuid4()) + ".bin")
        os.makedirs(model_path, exist_ok=True)
        with open(model_file, "wb") as f:
            f.write(model)
        model = h2o.load_model(os.path.abspath(model_file))
        test_frame = h2o.H2OFrame(X.to_pandas(), column_types=self.col_types)
        preds_frame = None

        try:
            if kwargs.get("pred_contribs"):
                return model.predict_contributions(test_frame).as_data_frame(header=False).values
            preds_frame = model.predict(test_frame)
            preds = preds_frame.as_data_frame(header=False)
            if self.num_classes == 1:
                return preds.values.ravel()
            elif self.num_classes == 2:
                return preds.iloc[:, -1].values.ravel()
            else:
                return preds.iloc[:, 1:].values
        finally:
            # h2o.remove(self.id) # Cannot remove id, do multiple predictions on same model
            h2o.remove(test_frame)
            remove(model_file)
            if preds_frame is not None:
                h2o.remove(preds_frame) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def predict(self, dataframe):
        import h2o

        predicted = self.h2o_model.predict(h2o.H2OFrame(dataframe)).as_data_frame()
        predicted.index = dataframe.index
        return predicted 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def h2o_iris_model():
    h2o.init()
    iris = datasets.load_iris()
    data = h2o.H2OFrame({
        'feature1': list(iris.data[:, 0]),
        'feature2': list(iris.data[:, 1]),
        'target': list(map(lambda i: "Flower %d" % i, iris.target))
    })
    train, test = data.split_frame(ratios=[.7])

    h2o_gbm = H2OGradientBoostingEstimator(ntrees=10, max_depth=6)
    h2o_gbm.train(['feature1', 'feature2'], 'target', training_frame=train)
    return ModelWithData(model=h2o_gbm, inference_data=test) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def process_h2o(X_train, X_test, y_train, df_types, m_type, seed,*args):
    """Function that trains and tests data using h2o's AutoML"""

    import h2o
    from h2o.automl import H2OAutoML

    ip = args[0] if len(args) > 0 else '127.0.0.1'
    port = np.random.randint(5555,8888)

    h2o.init(ip=ip, port=port, nthreads=N_CORES, min_mem_size=MIN_MEM, max_mem_size=MAX_MEM, ice_root='/tmp/')

    aml = None

    if(m_type == 'classification'):
        aml = H2OAutoML(max_runtime_secs=TIME_PER_TASK, seed=seed, sort_metric='AUTO')
    else:
        aml = H2OAutoML(max_runtime_secs=TIME_PER_TASK, seed=seed, sort_metric='MSE')

    dd = h2o.H2OFrame(pd.concat([X_train, y_train], axis=1))
    td = h2o.H2OFrame(X_test)

    # set categorical columns as 'factors'
    categ_cols = df_types[df_types['TYPE'] == 'categorical']['NAME'].values.tolist()
    if len(categ_cols) > 0:
        dd[categ_cols] = dd[categ_cols].asfactor()
        td[categ_cols] = td[categ_cols].asfactor()
    if m_type == 'classification':
        dd['target'] = dd['target'].asfactor()

    aml.train(y = 'target', training_frame = dd)
    response = aml.predict(td)
    pred =  (response[1:].as_data_frame().values if m_type == 'classification' else 
            response.as_data_frame().values.ravel())

    h2o.cluster().shutdown()

    return pred 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def _train_test_split_as_frames(x, y, is_str=False, is_classifier=False):
    y = y.astype(np.str) if is_str else y.astype(np.int64)
    x_train, x_test, y_train, _ = train_test_split(x, y, test_size=0.3, random_state=42)
    f_train_x = H2OFrame(x_train)
    f_train_y = H2OFrame(y_train)
    f_train = f_train_x.cbind(f_train_y)
    if is_classifier:
        f_train[f_train.ncol - 1] = f_train[f_train.ncol - 1].asfactor()
    return f_train, x_test.astype(np.float32) 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def predict_with_probabilities(self, data):
        data_frame = H2OFrame(data, column_names=self._column_names)
        preds = self._mojo_model.predict(data_frame).as_data_frame(use_pandas=True)
        if len(preds.columns) == 1:
            return [preds.to_numpy()]
        else:
            return [
                preds.iloc[:, 0].to_numpy().astype(np.str),
                preds.iloc[:, 1:].to_numpy()
            ] 

# 需要导入模块: import h2o [as 别名]
# 或者: from h2o import H2OFrame [as 别名]
def _prepare_one_hot(file, y, exclude_cols=None):
    if exclude_cols is None:
        exclude_cols = []
    dir_path = os.path.dirname(os.path.realpath(__file__))
    frame = h2o.import_file(dir_path + "/" + file)
    train, test = frame.split_frame([0.95], seed=42)

    cols_to_encode = []
    other_cols = []
    for name, ctype in test.types.items():
        if name == y or name in exclude_cols:
            pass
        elif ctype == "enum":
            cols_to_encode.append(name)
        else:
            other_cols.append(name)
    train_frame = train.as_data_frame()
    train_encode = train_frame.loc[:, cols_to_encode]
    train_other = train_frame.loc[:, other_cols + [y]]
    enc = OneHotEncoder(categories='auto', handle_unknown='ignore')
    enc.fit(train_encode)
    colnames = []
    for cidx in range(len(cols_to_encode)):
        for val in enc.categories_[cidx]:
            colnames.append(cols_to_encode[cidx] + "." + val)

    train_encoded = enc.transform(train_encode.values).toarray()
    train_encoded = pd.DataFrame(train_encoded)
    train_encoded.columns = colnames
    train = train_other.join(train_encoded)
    train = H2OFrame(train)

    test_frame = test.as_data_frame()
    test_encode = test_frame.loc[:, cols_to_encode]
    test_other = test_frame.loc[:, other_cols]

    test_encoded = enc.transform(test_encode.values).toarray()
    test_encoded = pd.DataFrame(test_encoded)
    test_encoded.columns = colnames
    test = test_other.join(test_encoded)

    return train, test