Python gridfs.GridFS方法代码示例

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def save_memory_to_db(self, env, name):
        try:
            env.logger.info(ArenaDDPGAgent.save_to_db, "Trying to save memory to database.")

            memory = pickle.dumps(self.memory)

            fs = gd.GridFS(env.db, collection=name+'_memory')

            fs.put(memory)

            del fs

            env.logger.info(ArenaDDPGAgent.save_to_db, "Memory saved to db!")

        except Exception as e:
            env.logger.error(ArenaDDPGAgent.save_to_db, env.parse_error(e)) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def get(self, _id, mode='r'):
            """Open the file with the specified id.

            .. warning::

                To avoid compatiblity issues, all files are
                opened in text-mode (`r`) by default, however
                for higher efficiency, files should generally
                be opened in binary mode (`rb`) whenever possible.

            :param _id: The file identifier.
            :type _id: str
            :param mode: The file mode used for opening.
            :returns: A file-like object to read from."""
            if mode == 'r':
                file = io.StringIO(self.gridfs.get(_id).read().decode())
                if len(file.getvalue()) > GRIDFS_LARGE_FILE_WARNING_THRSHLD:
                    warnings.warn(
                        "Open large GridFS files more efficiently in 'rb' mode.")
                return file
            elif mode == 'rb':
                return self.gridfs.get(file_id=_id)
            else:
                raise ValueError(mode) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def connect(self):
        """Connects to Mongo database, loads options and set connectors.
        @raise CuckooReportError: if unable to connect.
        """
        host = self.options.get("host", "127.0.0.1")
        port = self.options.get("port", 27017)
        db = self.options.get("db", "cuckoo")

        try:
            self.conn = MongoClient(host, port)
            self.db = self.conn[db]
            self.fs = GridFS(self.db)
        except TypeError:
            raise CuckooReportError("Mongo connection port must be integer")
        except ConnectionFailure:
            raise CuckooReportError("Cannot connect to MongoDB") 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def testCode4():
    inicio = 10569000
    client = MongoClient(envget('files.host'), envget('files.port'))
    db = client[envget('db_files_name')]
    fs = gridfs.GridFS(db)
    res = fs.find(timeout=False).skip(inicio)
    lc = Launcher()
    count = inicio
    reset = 0
    for f in res:
        data = f.read()
        # print(f.filename,count)
        lc.launchFileAnalitics((f.filename, data))
        reset += 1
        count += 1
        if(reset >= 1000):
            print(str(count) + " processed")
            reset = 0
    print(str(count) + " processed")

# ---------------------------------------------- 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def remove(self, workflow_id):
        """ Removes a document specified by its id from the data store.

        All associated GridFs documents are deleted as well.

        Args:
            workflow_id (str): The id of the document that represents a workflow run.

        Raises:
            DataStoreNotConnected: If the data store is not connected to the server.
        """
        try:
            db = self._client[self.database]
            fs = GridFSProxy(GridFS(db.unproxied_object))

            for grid_doc in fs.find({"workflow_id": workflow_id},
                                    no_cursor_timeout=True):
                fs.delete(grid_doc._id)

            col = db[WORKFLOW_DATA_COLLECTION_NAME]
            return col.delete_one({"_id": ObjectId(workflow_id)})

        except ConnectionFailure:
            raise DataStoreNotConnected() 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def get(self, workflow_id):
        """ Returns the document for the given workflow id.

        Args:
            workflow_id (str): The id of the document that represents a workflow run.

        Raises:
            DataStoreNotConnected: If the data store is not connected to the server.

        Returns:
            DataStoreDocument: The document for the given workflow id.
        """
        try:
            db = self._client[self.database]
            fs = GridFSProxy(GridFS(db.unproxied_object))
            return DataStoreDocument(db[WORKFLOW_DATA_COLLECTION_NAME], fs, workflow_id)

        except ConnectionFailure:
            raise DataStoreNotConnected() 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def _encode_value(self, value):
        """ Encodes the value such that it can be stored into MongoDB.

        Any primitive types are stored directly into MongoDB, while non-primitive types
        are pickled and stored as GridFS objects. The id pointing to a GridFS object
        replaces the original value.

        Args:
            value (object): The object that should be encoded for storing in MongoDB.

        Returns:
            object: The encoded value ready to be stored in MongoDB.
        """
        if isinstance(value, (int, float, str, bool, datetime)):
            return value
        elif isinstance(value, list):
            return [self._encode_value(item) for item in value]
        elif isinstance(value, dict):
            result = {}
            for key, item in value.items():
                result[key] = self._encode_value(item)
            return result
        else:
            return self._gridfs.put(Binary(pickle.dumps(value)),
                                    workflow_id=self._workflow_id) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def __init__(self):
        if MONGO:
            # Create the connection
            mongo_uri = config.TannerConfig.get('MONGO', 'URI')

            connection = pymongo.MongoClient(mongo_uri)

            # Connect to Databases.
            tandb = connection['tanner']
            tandbfs = connection['voldbfs']

            # Get Collections
            self.tan_sessions = tandb.sessions
            self.tan_files = GridFS(tandbfs)

            # Indexes
            self.tan_sessions.create_index([('$**', 'text')])
        else:
            print('pymongo not found. pip install pymongo') 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def update_plugin(self, plugin_id, new_values):
        plugin_id = ObjectId(plugin_id)
        if len(str(new_values)) > 12000000:
            print "Storing Large Document in GridFS"
            large_document = json.dumps(new_values['plugin_output'])
            large_document_id = self.create_file(large_document, 'sess_id', 'sha256', 'filename', pid=None, file_meta=None)
            new_values['plugin_output'] = large_document_id
            new_values['largedoc'] = 'True'

        self.vol_plugins.update_one({'_id': plugin_id}, {"$set": new_values})
        return True


    ##
    # File System
    ## 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def mongo_obs(monkeypatch):
    client = ReconnectingMongoClient(
        max_calls_before_reconnect=10,
        max_calls_before_failure=1,
        exception_to_raise=pymongo.errors.ServerSelectionTimeoutError,
    )
    fs = gridfs.GridFS(client.sacred)
    monkeypatch.setattr(pymongo, "MongoClient", lambda *args, **kwargs: client)
    monkeypatch.setattr(gridfs, "GridFS", lambda _: fs)

    return QueuedMongoObserver(interval=0.01, retry_interval=0.01) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def mongo_obs():
    db = mongomock.MongoClient().db
    runs = db.runs
    metrics = db.metrics
    fs = gridfs.GridFS(db)
    return MongoObserver.create_from(runs, fs, metrics_collection=metrics) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def failing_mongo_observer():
    db = FailingMongoClient(
        max_calls_before_failure=2,
        exception_to_raise=pymongo.errors.ServerSelectionTimeoutError,
    ).db

    runs = db.runs
    metrics = db.metrics
    fs = gridfs.GridFS(db)
    return MongoObserver.create_from(runs, fs, metrics_collection=metrics) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def __init__(self, client, db='retelogs', collection=None):
        self.client = MongoClient(client)
        self.db = self.client[db]
        self.collection = collection or 'pictures'
        self.fs = gridfs.GridFS(self.db, collection) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def load_memory_from_db(self, env, number=-1):
        try:
            col_names = env.db.collection_names()

            if self.name + '_memory' + '.chunks' not in col_names:

                print("Choice the table number containing the memory buffer to be loaded:\n")
                for i, table in enumerate(env.db.collection_names()):
                    print("Table {2}: {0}, count: {1}".format(table, env.db[table].count(), i))
                table_n = input("Enter the table number or press ENTER:")
                if not table_n:
                    print("Load memory skiped.")
                    return
                elif table_n == '':
                    print("Load memory skiped.")
                    return
                else:
                    memory_name = col_names[int(table_n)].strip('.chunks')
            else:
                memory_name = self.name + '_memory'

            env.logger.info(ArenaDDPGAgent.load_from_db, "Trying to read data from " + memory_name)

            fs = gd.GridFS(env.db, collection=memory_name)

            memory = pickle.loads(fs.find().sort('uploadDate', -1).skip(abs(number)-1).limit(-1).next().read())

            del fs

            self.memory = memory

            env.logger.info(ArenaDDPGAgent.load_from_db, "Memory buffer loaded from " + memory_name)

        except Exception as e:
            env.logger.error(ArenaDDPGAgent.load_memory_from_db, env.parse_error(e)) 

# 需要导入模块: import gridfs [as 别名]
# 或者: from gridfs import GridFS [as 别名]
def save_to_db(self, env, name, overwrite=False):
        try:
            timestamp = datetime.utcnow()

            env.logger.info(ArenaDDPGAgent.save_to_db, "Trying to save model to database. Weights version: %d" %
                            (self.weights_version))

            data = {'index': timestamp,
                    'weights_version': self.weights_version + 1,
                    # 'agent_params': pickle.dumps(agent_params),
                    'actor_config': pickle.dumps(self.actor.to_json()),
                    'critic_config': pickle.dumps(self.critic.to_json()),
                    'actor_weights': pickle.dumps(self.actor.get_weights()),
                    'critic_weights': pickle.dumps(self.critic.get_weights()),
                    # 'memory': pickle.dumps(self.memory)
                    }

            fs = gd.GridFS(env.db, collection=name)

            fs.put(pickle.dumps(data))

            del fs

            env.logger.info(ArenaDDPGAgent.save_to_db, "Model weights saved to db!")

        except Exception as e:
            env.logger.error(ArenaDDPGAgent.save_to_db, env.parse_error(e))