Python cloudpickle.dump方法代码示例

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save(self, path=None):
        """Save model to a pickle located at `path`"""
        if path is None:
            path = os.path.join(logger.get_dir(), "model.pkl")

        with tempfile.TemporaryDirectory() as td:
            save_state(os.path.join(td, "model"))
            arc_name = os.path.join(td, "packed.zip")
            with zipfile.ZipFile(arc_name, 'w') as zipf:
                for root, dirs, files in os.walk(td):
                    for fname in files:
                        file_path = os.path.join(root, fname)
                        if file_path != arc_name:
                            zipf.write(file_path, os.path.relpath(file_path, td))
            with open(arc_name, "rb") as f:
                model_data = f.read()
        with open(path, "wb") as f:
            cloudpickle.dump((model_data, self._act_params), f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save_act(self, path=None):
        """Save model to a pickle located at `path`"""
        if path is None:
            path = os.path.join(logger.get_dir(), "model.pkl")

        with tempfile.TemporaryDirectory() as td:
            save_state(os.path.join(td, "model"))
            arc_name = os.path.join(td, "packed.zip")
            with zipfile.ZipFile(arc_name, 'w') as zipf:
                for root, dirs, files in os.walk(td):
                    for fname in files:
                        file_path = os.path.join(root, fname)
                        if file_path != arc_name:
                            zipf.write(file_path, os.path.relpath(file_path, td))
            with open(arc_name, "rb") as f:
                model_data = f.read()
        with open(path, "wb") as f:
            cloudpickle.dump((model_data, self._act_params), f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = StringIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save(self, directory: str):
        shutil.copyfile(self.model_file, os.path.join(directory, 'rnn.pt'))
        shell(f'rm -f {self.model_file}')
        with open(os.path.join(directory, 'rnn.pkl'), 'wb') as f:
            cloudpickle.dump({
                'page_field': self.page_field,
                'text_field': self.text_field,
                'qanta_id_field': self.qanta_id_field,
                'n_classes': self.n_classes,
                'gradient_clip': self.gradient_clip,
                'n_hidden_units': self.n_hidden_units,
                'n_hidden_layers': self.n_hidden_layers,
                'nn_dropout': self.nn_dropout,
                'batch_size': self.batch_size,
                'use_wiki': self.use_wiki,
                'n_wiki_sentences': self.n_wiki_sentences,
                'wiki_title_replace_token': self.wiki_title_replace_token,
                'lowercase': self.lowercase,
                'random_seed': self.random_seed,
                'config_num': self.config_num
            }, f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save_policy(self, policy_path: str):
        """Save a policy to a pickle. Can be reloaded by `.reconstruct_policy()`.

        Args:
            policy_path: path to save policy to.
        """
        policy_params = self.sess.run(self.policy_variables)
        data = {
            "class": self.policy_class,
            "kwargs": self.policy_kwargs,
            "params": policy_params,
        }
        dirname = os.path.dirname(policy_path)
        if dirname:
            os.makedirs(dirname, exist_ok=True)
        with open(policy_path, "wb") as fp:
            cloudpickle.dump(data, fp) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def _save_to_file_cloudpickle(save_path, data=None, params=None):
        """Legacy code for saving models with cloudpickle

        :param save_path: (str or file-like) Where to store the model
        :param data: (OrderedDict) Class parameters being stored
        :param params: (OrderedDict) Model parameters being stored
        """
        if isinstance(save_path, str):
            _, ext = os.path.splitext(save_path)
            if ext == "":
                save_path += ".pkl"

            with open(save_path, "wb") as file_:
                cloudpickle.dump((data, params), file_)
        else:
            # Here save_path is a file-like object, not a path
            cloudpickle.dump((data, params), save_path) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save_act(self, path=None):
        """Save model to a pickle located at `path`"""
        if path is None:
            path = os.path.join(logger.get_dir(), "model.pkl")

        with tempfile.TemporaryDirectory() as td:
            save_variables(os.path.join(td, "model"))
            arc_name = os.path.join(td, "packed.zip")
            with zipfile.ZipFile(arc_name, 'w') as zipf:
                for root, dirs, files in os.walk(td):
                    for fname in files:
                        file_path = os.path.join(root, fname)
                        if file_path != arc_name:
                            zipf.write(file_path, os.path.relpath(file_path, td))
            with open(arc_name, "rb") as f:
                model_data = f.read()
        with open(path, "wb") as f:
            cloudpickle.dump((model_data, self._act_params), f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def _save_candidate_classes(candidate_classes: List[Candidate], path: str) -> None:
    pickle.dump(
        [
            {
                "class_name": candidate_class.__name__,
                "mention_class_names": [
                    candidate_class.__name__
                    for candidate_class in candidate_class.mentions
                ],
                "table_name": candidate_class.__tablename__,
                "cardinality": candidate_class.cardinality,
                "values": candidate_class.values,
            }
            for candidate_class in candidate_classes
        ],
        open(os.path.join(path, "candidate_classes.pkl"), "wb"),
    ) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = BytesIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save(self, path=None):
    """Save model to a pickle located at `path`"""
    if path is None:
        path = os.path.join(logger.get_dir(), "model.pkl")

    with tempfile.TemporaryDirectory() as td:
        U.save_state(os.path.join(td, "model"))
        arc_name = os.path.join(td, "packed.zip")
        with zipfile.ZipFile(arc_name, 'w') as zipf:
            for root, dirs, files in os.walk(td):
                for fname in files:
                    file_path = os.path.join(root, fname)
                    if file_path != arc_name:
                        zipf.write(file_path, os.path.relpath(file_path, td))
        with open(arc_name, "rb") as f:
            model_data = f.read()
    with open(path, "wb") as f:
        cloudpickle.dump((model_data), f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def _save_custom_objects(path, custom_objects):
    """
    Save custom objects dictionary to a cloudpickle file so a model can be easily loaded later.

    :param path: An absolute path that points to the data directory within /path/to/model.
    :param custom_objects: Keras ``custom_objects`` is a dictionary mapping
                           names (strings) to custom classes or functions to be considered
                           during deserialization. MLflow saves these custom layers using
                           CloudPickle and restores them automatically when the model is
                           loaded with :py:func:`mlflow.keras.load_model` and
                           :py:func:`mlflow.pyfunc.load_model`.
    """
    import cloudpickle
    custom_objects_path = os.path.join(path, _CUSTOM_OBJECTS_SAVE_PATH)
    with open(custom_objects_path, "wb") as out_f:
        cloudpickle.dump(custom_objects, out_f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def _save_model(sk_model, output_path, serialization_format):
    """
    :param sk_model: The scikit-learn model to serialize.
    :param output_path: The file path to which to write the serialized model.
    :param serialization_format: The format in which to serialize the model. This should be one of
                                 the following: ``mlflow.sklearn.SERIALIZATION_FORMAT_PICKLE`` or
                                 ``mlflow.sklearn.SERIALIZATION_FORMAT_CLOUDPICKLE``.
    """
    with open(output_path, "wb") as out:
        if serialization_format == SERIALIZATION_FORMAT_PICKLE:
            pickle.dump(sk_model, out)
        elif serialization_format == SERIALIZATION_FORMAT_CLOUDPICKLE:
            import cloudpickle
            cloudpickle.dump(sk_model, out)
        else:
            raise MlflowException(
                    message="Unrecognized serialization format: {serialization_format}".format(
                        serialization_format=serialization_format),
                    error_code=INTERNAL_ERROR) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save(self, path=None):
        """Save model to a pickle located at `path`"""
        if path is None:
            path = os.path.join(logger.get_dir(), "model.pkl")

        with tempfile.TemporaryDirectory() as td:
            U.save_state(os.path.join(td, "model"))
            arc_name = os.path.join(td, "packed.zip")
            with zipfile.ZipFile(arc_name, 'w') as zipf:
                for root, dirs, files in os.walk(td):
                    for fname in files:
                        file_path = os.path.join(root, fname)
                        if file_path != arc_name:
                            zipf.write(file_path, os.path.relpath(file_path, td))
            with open(arc_name, "rb") as f:
                model_data = f.read()
        with open(path, "wb") as f:
            cloudpickle.dump((model_data, self._act_params), f) 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def dump(self, obj):
        self.inject_addons()
        try:
            return Pickler.dump(self, obj)
        except RuntimeError as e:
            if 'recursion' in e.args[0]:
                msg = """Could not pickle object as excessively deep recursion required."""
                raise pickle.PicklingError(msg)
            else:
                raise 

# 需要导入模块: import cloudpickle [as 别名]
# 或者: from cloudpickle import dump [as 别名]
def save(self, directory: str) -> None:
        shutil.copyfile(self.model_file, os.path.join(directory, 'elmo.pt'))
        shell(f'rm -f {self.model_file}')
        with open(os.path.join(directory, 'elmo.pkl'), 'wb') as f:
            cloudpickle.dump({
                'class_to_i': self.class_to_i,
                'i_to_class': self.i_to_class,
                'config_num': self.config_num,
                'random_seed': self.random_seed,
                'dropout': self.dropout
            }, f)