Python joblib.dump方法代码示例

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save_model(self):
        """
        Saves all necessary model state information for classification work to disk.
        :return: True if it succeeded and False otherwise.
        """
        # if we aren't keeping the extracted file details to reproduce the analysis, let's clear that data and
        # save the model.  It's not needed to perform basic predictions on new files.
        if self.retain_sample_contents is False:
            metadata = {'filemodified', 'extracted_vba', 'filename_vba', 'filepath', 'filename', 'function_names',
                        'filesize', 'filemodified', 'stream_path'}
            metadata_delete = list(metadata & set(self.modeldata.columns))
            self.modeldata.drop(metadata_delete, axis=1, inplace=True)

        try:
            saved_model = {'modeldata': self.modeldata,
                           'features': self.features,
                           'model_cntvect_cnts_array': self.modeldata_cnts.toarray()
                           }

            joblib.dump(saved_model, self.modeldata_pickle)

        except Exception as e:
            raise IOError("Error saving model data to disk: {}".format(str(e)))
            return False
        return True 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save(self, filepath):
        joblib.dump(self, filepath, 3) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save_to_disk(dataset, filename, compress=3):
  """Save a dataset to file."""
  if filename.endswith('.joblib'):
    joblib.dump(dataset, filename, compress=compress)
  elif filename.endswith('.npy'):
    np.save(filename, dataset)
  else:
    raise ValueError("Filename with unsupported extension: %s" % filename) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save_metadata(tasks, metadata_df, data_dir):
  """
  Saves the metadata for a DiskDataset
  Parameters
  ----------
  tasks: list of str
    Tasks of DiskDataset
  metadata_df: pd.DataFrame
  data_dir: str
    Directory to store metadata
  Returns
  -------
  """
  if isinstance(tasks, np.ndarray):
    tasks = tasks.tolist()
  metadata_filename = os.path.join(data_dir, "metadata.csv.gzip")
  tasks_filename = os.path.join(data_dir, "tasks.json")
  with open(tasks_filename, 'w') as fout:
    json.dump(tasks, fout)
  metadata_df.to_csv(metadata_filename, index=False, compression='gzip') 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def _tf_simple_save(self, itr=None):
        """
        Uses simple_save to save a trained model, plus info to make it easy
        to associated tensors to variables after restore. 
        """
        if proc_id()==0:
            assert hasattr(self, 'tf_saver_elements'), \
                "First have to setup saving with self.setup_tf_saver"
            fpath = 'tf1_save' + ('%d'%itr if itr is not None else '')
            fpath = osp.join(self.output_dir, fpath)
            if osp.exists(fpath):
                # simple_save refuses to be useful if fpath already exists,
                # so just delete fpath if it's there.
                shutil.rmtree(fpath)
            tf.saved_model.simple_save(export_dir=fpath, **self.tf_saver_elements)
            joblib.dump(self.tf_saver_info, osp.join(fpath, 'model_info.pkl')) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def create_model_from_signatures(sig_csv_path, model_out, sig_datatype=np.int32):
    """
    Takes a .csv file containing class signatures - produced by extract_features_to_csv - and uses it to train
    and pickle a scikit-learn model.

    Parameters
    ----------
    sig_csv_path
        The path to the signatures file
    model_out
        The location to save the pickled model to.
    sig_datatype
        The datatype to read the csv as. Defaults to int32.

    Notes
    -----
    At present, the model is an ExtraTreesClassifier arrived at by tpot:
    model = ens.ExtraTreesClassifier(bootstrap=False, criterion="gini", max_features=0.55, min_samples_leaf=2,
                                 min_samples_split=16, n_estimators=100, n_jobs=4, class_weight='balanced')
    """
    model = ens.ExtraTreesClassifier(bootstrap=False, criterion="gini", max_features=0.55, min_samples_leaf=2,
                                     min_samples_split=16, n_estimators=100, n_jobs=4, class_weight='balanced')
    features, labels = load_signatures(sig_csv_path, sig_datatype)
    model.fit(features, labels)
    joblib.dump(model, model_out) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save_itr_params(itr, params, prefix='', save_anyway=False):
    if _snapshot_dir:
        if len(prefix) > 0:
            prefix = prefix + '_'
        if _snapshot_mode == 'all':
            file_name = osp.join(_snapshot_dir, prefix + 'itr_%d.pkl' % itr)
            pickle.dump(params, open(file_name, "wb"))
        elif _snapshot_mode == 'last':
            # override previous params
            file_name = osp.join(_snapshot_dir, prefix + 'params.pkl')
            pickle.dump(params, open(file_name, "wb"))
        elif _snapshot_mode == "gap":
            if save_anyway or itr % _snapshot_gap == 0:
                file_name = osp.join(_snapshot_dir, prefix + 'itr_%d.pkl' % itr)
                pickle.dump(params, open(file_name, "wb"))
        elif _snapshot_mode == "gap_and_last":
            if save_anyway or itr % _snapshot_gap == 0:
                file_name = osp.join(_snapshot_dir, prefix + 'itr_%d.pkl' % itr)
                pickle.dump(params, open(file_name, "wb"))
            file_name = osp.join(_snapshot_dir, prefix + 'params.pkl')
            pickle.dump(params, open(file_name, "wb"))
        elif _snapshot_mode == 'none':
            pass
        else:
            raise NotImplementedError 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def main():
    args = parse_args()

    features_extractor = FaceFeaturesExtractor()
    embeddings, labels, class_to_idx = load_data(args, features_extractor)
    clf = train(args, embeddings, labels)

    idx_to_class = {v: k for k, v in class_to_idx.items()}

    target_names = map(lambda i: i[1], sorted(idx_to_class.items(), key=lambda i: i[0]))
    print(metrics.classification_report(labels, clf.predict(embeddings), target_names=list(target_names)))

    if not os.path.isdir(MODEL_DIR_PATH):
        os.mkdir(MODEL_DIR_PATH)
    model_path = os.path.join('model', 'face_recogniser.pkl')
    joblib.dump(FaceRecogniser(features_extractor, clf, idx_to_class), model_path) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def save(self, filename, ensure_compatibility = True):
        """
        Pickle a class instance. E.g., corex.save('saved.pkl')
        When set to True, ensure_compatibility resets self.words before saving
        a pickle to avoid Unicode loading issues usually seen when trying to load
        the pickle from a Python 2 implementation.
        It is recommended to set it to False if you know you are going to load the
        model in an all Python 3 implementation as self.words is required for fetching
        the topics via get_topics().
        """
        # Avoid saving words with object.
        #TODO: figure out why Unicode sometimes causes an issue with loading after pickling
        temp_words = self.words
        if ensure_compatibility and (self.words is not None):
            self.words = None

        # Save CorEx object
        import pickle
        if path.dirname(filename) and not path.exists(path.dirname(filename)):
            makedirs(path.dirname(filename))
        pickle.dump(self, open(filename, 'wb'), protocol=-1)
        # Restore words to CorEx object
        self.words = temp_words 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def fit_log_reg(X, y):
    # fits a logistic regression model to your data
    model = LogisticRegression(class_weight='balanced')
    model.fit(X, y)
    print('Train size: ', len(X))
    train_score = model.score(X, y)
    print('Training accuracy', train_score)
    ypredz = model.predict(X)
    cm = confusion_matrix(y, ypredz)
    # tn, fp, fn, tp = cm.ravel()
    tn, _, _, tp = cm.ravel()

    # true positive rate When it's actually yes, how often does it predict yes?
    recall = float(tp) / np.sum(cm, axis=1)[1]
    # Specificity: When it's actually no, how often does it predict no?
    specificity = float(tn) / np.sum(cm, axis=1)[0]

    print('Recall/ Like accuracy', recall)
    print('specificity/ Dislike accuracy', specificity)

    # save the model
    joblib.dump(model, 'log_reg_model.pkl') 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def write_to_file(obj, filename, path=None, overwrite=False):
    if path is not None:
        filename = os.path.join(path, filename)
    filename = os.path.abspath(filename)
    output_dir = os.path.dirname(filename)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if not overwrite and os.path.exists(filename):
        print("WARNING: file already exists %s; not overwriting." % (filename,))
        pass
        # Check to see whether same as one on disk?
        # When to overwrite?
    else:
        print("Writing to %s" % (filename,))
        joblib.dump(obj, filename)


# Special-case stuff
# ------------------ 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def test_model_joblib_serialization(teardown, dump, load):
    x_data = iris.data
    y_t_data = iris.target
    random_state = 123
    n_components = 2

    stacked_model_baikal = make_naive_stacked_model(
        n_components, random_state, x_data, y_t_data
    )
    y_pred_baikal = stacked_model_baikal.predict(x_data)

    # Persist model to a file
    f = tempfile.TemporaryFile()
    dump(stacked_model_baikal, f)
    f.seek(0)
    stacked_model_baikal_2 = load(f)
    y_pred_baikal_2 = stacked_model_baikal_2.predict(x_data)

    assert_array_equal(y_pred_baikal_2, y_pred_baikal) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def read_grid_pkl(tmpdir):
    expected = {'lon_min_x': 116.319236,
                'lat_min_y': 39.984094,
                'grid_size_lat_y': 5,
                'grid_size_lon_x': 5,
                'cell_size_by_degree': 0.0001353464801860623
                }
    d = tmpdir.mkdir('core')

    file_write_default = d.join('test_read_grid.pkl')
    filename_write_default = os.path.join(
        file_write_default.dirname, file_write_default.basename
    )

    grid = _default_grid()

    with open(filename_write_default, 'wb') as f:
        joblib.dump(grid.get_grid(), f)

    saved_grid = grid.read_grid_pkl(filename_write_default)

    assert_equal(saved_grid, expected) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def test_modelpipeline_pickling_preserves_template_ids(
        version, train_id, predict_id):
    # Test that pickling a ModelPipeline object preserves the template IDs
    # that have already been set during object instantiation.
    with TemporaryDirectory() as temp_dir:
        mp = _model.ModelPipeline('wf', 'dv', civisml_version=version)

        # Before pickling, make sure the template IDs are set as expected
        assert mp.train_template_id == train_id
        assert mp.predict_template_id == predict_id

        pickle_path = os.path.join(temp_dir, 'model.pkl')

        with open(pickle_path, 'wb') as f:
            pickle.dump(mp, f)

        with open(pickle_path, 'rb') as f:
            mp_unpickled = pickle.load(f)

        # After unpickling, the template IDs should remain.
        assert mp_unpickled.train_template_id == train_id
        assert mp_unpickled.predict_template_id == predict_id 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import dump [as 别名]
def _tf_simple_save(self, itr=None):
        """
        Uses simple_save to save a trained model, plus info to make it easy
        to associated tensors to variables after restore. 
        """
        if proc_id()==0:
            assert hasattr(self, 'tf_saver_elements'), \
                "First have to setup saving with self.setup_tf_saver"
            fpath = 'simple_save' + ('%d'%itr if itr is not None else '')
            fpath = osp.join(self.output_dir, fpath)
            if osp.exists(fpath):
                # simple_save refuses to be useful if fpath already exists,
                # so just delete fpath if it's there.
                shutil.rmtree(fpath)
            tf.saved_model.simple_save(export_dir=fpath, **self.tf_saver_elements)
            joblib.dump(self.tf_saver_info, osp.join(fpath, 'model_info.pkl'))