Python Store.open方法代码示例

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import open [as 别名]
class NodeServer(object):
    def __init__(self, node_id, store_file=None, explicit_configuration=None, coordinator_addresses=[], var_directory='var',
                 configuration_update_interval=300):
        super(NodeServer, self).__init__()
        if __debug__: logging.debug('node_id = %s, store_file = %s, explicit_configuration = %s, coordinators = %s, var_directory = %s',
                  node_id, store_file, explicit_configuration, coordinator_addresses, var_directory)
        self.id = node_id
        self.node = None
        var_directory = paths.path(var_directory)
        store_file = store_file or os.path.join(var_directory, 'data', '%s.tcb' % self.id)
        self.__store = Store(store_file)
        self.__store.open()
        self.__node_clients = {}
        self.__internal_cluster_client = service.MulticastClient(InternalNodeServiceProtocol())
        configuration_directory = os.path.join(var_directory, 'etc')
        self.__repair_task = None
        self.__background_repair_enabled = False
        self.__background_repair_interval_seconds = None
        self.__read_repair_enabled = False
        self.__configuration = None
        self.__configuration_controller = ConfigurationController('nodeserver-%s' % self.id,
                                                                  coordinator_addresses, explicit_configuration,
                                                                  configuration_directory, self.__update_configuration,
                                                                  configuration_update_interval)


    def __initialize_node_client_pool(self):
        nodes = self.__configuration.find_neighbour_nodes_for_node(self.node) if self.node else {}

        recycled_node_clients = {}
        for node_id, client in self.__node_clients.iteritems():
            if node_id in nodes:
                recycled_node_clients[node_id] = client
            else:
                client.disconnect()

        new_node_clients = {}
        for node_id, node in nodes.iteritems():
            if node_id not in recycled_node_clients:
                new_node_clients[node_id] = service.Client((node.address, node.port), InternalNodeServiceProtocol, tag=node_id)

        self.__node_clients = dict(recycled_node_clients, **new_node_clients)

        # Blocking, do this after setting self.__node_clients
        for client in new_node_clients.itervalues():
            client.connect()

    def __update_configuration(self, new_configuration):
        if __debug__: logging.debug('New configuration: %s', new_configuration)
        self.__configuration = new_configuration
        deployment = None
        if self.id in new_configuration.active_deployment.nodes:
            deployment = new_configuration.active_deployment
        if new_configuration.target_deployment and self.id in new_configuration.target_deployment.nodes:
            deployment = new_configuration.target_deployment
        self.node = deployment.nodes.get(self.id, None) if deployment else None
        self.__read_repair_enabled = deployment.read_repair_enabled if deployment else False
        self.__background_repair_enabled = deployment.background_repair_enabled if deployment else False
        self.__background_repair_interval_seconds = deployment.background_repair_interval_seconds if deployment else 0
        self.__initialize_node_client_pool()
        # TODO: restart servers if addresses changed

    def __fetch_value(self, key, node_id):
        if __debug__: logging.debug('key: %s, node_id: %s', key, node_id)
        return self.__clients_for_nodes((node_id,))[0].get(key) or (None, None)

    def __fetch_timestamps(self, key, node_ids):
        if __debug__: logging.debug('key: %s', key)
        node_ids = dict(node_ids)
        node_ids.pop(self.node.id, None)
        if not node_ids:
            return []
        clients = self.__clients_for_nodes(node_ids)
        return self.__internal_cluster_client.stat(clients, key)

    def __clients_for_nodes(self, node_ids):
        return [self.__node_clients[node_id] for node_id in node_ids]

    def __propagate(self, key, timestamp, value, target_nodes):
        if __debug__: logging.debug('key: %s, target_nodes: %s', key, target_nodes)
        collector = self.__internal_cluster_client.set_collector(self.__clients_for_nodes(target_nodes), 1)
        self.__internal_cluster_client.set_async(collector, key, timestamp, value)

    def __read_repair(self, key, timestamp, value, node_ids):
        if __debug__: logging.debug('key: %s, timestamp: %s', key, timestamp)
        remote_timestamps = self.__fetch_timestamps(key, node_ids)
        if __debug__: logging.debug('remote: %s', [(client.tag, repr(remote_timestamp)) for client, remote_timestamp in remote_timestamps])
        newer = [(client, remote_timestamp) for client, remote_timestamp in remote_timestamps
                 if remote_timestamp and remote_timestamp > timestamp]

        if __debug__: logging.debug('newer: %s', [(client.tag, repr(remote_timestamp)) for client, remote_timestamp in newer])
        if newer:
            latest_client, latest_timestamp = newer[-1]
            latest_timestamp, latest_value = self.__fetch_value(key, latest_client.tag)
            if __debug__: logging.debug('latest_timestamp: %s', latest_timestamp)
            if latest_timestamp and latest_value:
                value = latest_value
                timestamp = latest_timestamp

        older = [(client, remote_timestamp) for client, remote_timestamp in remote_timestamps
#.........这里部分代码省略.........

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import open [as 别名]
class DP:
    # list of authors that are remembered
    author_list = []
    # list of authors to skip crawling for various reasons
    skip_list = ['[deleted]']
    # internal sqlite3 store
    store = None
    
    def write_to_queue(self, data, prefix='tmp'):
        fh, filename = tempfile.mkstemp(dir=os.path.join(tmpdir, 'dp', 'queue'), prefix=prefix)
        os.close(fh)
        fp = open(filename, 'w')
        fp.write(data)
        fp.close()
        return os.path.split(filename)[1]

    def seed(self):
        self.store = Store('/collection/sharvey/reddit/')
        self.store.open()
        print 'Created seed queue'
        return self.write_to_queue('a,t3_1u4kuf', 'tmp_a_')

    def process_author(self, abspath, filename):
        filetype = filename.split('_')
        fp = open(os.path.join(abspath, filename))
        blob = json.load(fp)
        fp.close()
        elements = parser.extract_listing_elements(blob)
        self.store.store_author(elements)
        return []

    def process_snapshot(self, abspath, filename):
        filetype = filename.split('_')
        fp = open(os.path.join(abspath, filename))
        blob = json.load(fp)
        fp.close()
        if filetype[0] == 'a':
            posts = blob['posts']
            nav = blob['nav']
            start_hit = False
            queue_file_list = []
            queue_list = []
            for sube in posts:
                utctime = int(sube['created_utc'])
                sttime = time.strftime('%Y%m%d', time.gmtime(utctime))
                if (int(sttime) > int(dateend)):
                    continue
                elif (int(sttime) < int(datestart)):
                    start_hit = True
                    break
                else:
                    queue_list.append('p,'+sube['id'])
            queue_file_list.append(self.write_to_queue('\n'.join(queue_list), 'tmp_p_'))
            if start_hit is not True:
                if nav['after'] is not None:
                    queue_file_list.append(self.write_to_queue('a,'+nav['after'], 'tmp_a_'))
            return queue_file_list
        elif filetype[0] == 'p':
            post = blob['post']
            comments = blob['comments']
            self.store.store_snapshot(post, comments)
            
            if crawl_author:
                queue_file_list = []
                if post['author'] not in self.author_list and post['author'] not in self.skip_list:
                    queue_file_list.append(self.write_to_queue('u,'+post['author'], 'tmp_u_'))
                    self.author_list.append(post['author'])
                for comment in comments:
                    if comment['author'] not in self.author_list and comment['author'] not in self.skip_list:
                        queue_file_list.append(self.write_to_queue('u,'+comment['author'], 'tmp_u_'))
                        self.author_list.append(comment['author'])
                return queue_file_list
        return []

    def process_snapshots(self, abspath, filename_list):
        post_tuples = []
        for filename in filename_list:
            filetype = filename.split('_')
            fp = open(os.path.join(abspath, filename))
            blob = json.load(fp)
            fp.close()
            post_tuples.append( (blob['post'], blob['comments']) )
        self.store.store_batch_snapshot(post_tuples)
            
        if crawl_author:
                queue_file_list = []

    def run(self):
        seedfile = self.seed()
        os.rename(os.path.join(tmpdir, 'dp', 'queue', seedfile), os.path.join(tmpdir, 'server', 'queue', seedfile))
        sleepcount = 0
        while True:
            for filename in os.listdir(os.path.join(tmpdir, 'dp', 'staging')):
                sleepcount = 0
                self.store.open()
                prefix = filename.split('.')[0]
                absfilename = os.path.join(tmpdir, 'dp', 'staging', filename)
                abspath = os.path.join(tmpdir, 'dp', 'staging', prefix)

                os.mkdir(abspath)
#.........这里部分代码省略.........