Python Ring.get_part_nodes方法代码示例

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]
def is_valid_ring(ring_file):
    """Check if a ring file is 'valid'
        - make sure it has more than one device
        - make sure get_part_nodes works
    :returns: True or False if ring is valid
    """
    try:
        ring = Ring(ring_file)
        if len(ring.devs) < 1:
            return False
        if not ring.get_part_nodes(1):
            return False
    except Exception:
        return False
    return True

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]
class SwiftUsageInput(IDataSource):
    def __init__(self, project, username, password, auth_url, ring_location="/etc/swift/account.ring.gz"):
        self.__project=project
        self.__username=username
        self.__password=password
        self.__auth_url=auth_url
        self.__ring_location=ring_location
        self.__ring=None
    def get_data(self, **kwargs):
        query_id=str(uuid.uuid4())
        data_list=[]
        from keystoneclient.v2_0 import client
        from swift.common.ring import Ring
        keystone=client.Client(username=self.__username, password=self.__password, tenant_name=self.__project, auth_url=self.__auth_url)
        self.__ring = Ring(self.__ring_location)
        tenants = [tenant.id for tenant in keystone.tenants.list()]
        random.shuffle(tenants)
        data=init_message()
        data["swift"] = {}
        for tenant, stats in zip(tenants, ThreadPool().map(self.fetch, tenants)):
            if stats is not None:
                data["swift"][tenant] = stats
        return data
        
    def fetch(self, tenant):
        account = "AUTH_%s" % tenant
        partition = self.__ring.get_part(account, None, None)
        nodes = self.__ring.get_part_nodes(partition)
        random.shuffle(nodes)
        for node in nodes:
            url = "http://%s:%s/%s/%s/%s" % (node["ip"], node["port"], node["device"], partition, account)
            try:
                response = requests.head(url, timeout=5)
                if response.status_code == 204:
                    return {
                        "containers" : int(response.headers["x-account-container-count"]),
                        "objects" : int(response.headers["x-account-object-count"]),
                        "bytes" : int(response.headers["x-account-bytes-used"]),
                        "quota" : int(response.headers["x-account-meta-quota-bytes"]) if "x-account-meta-quota-bytes" in response.headers else None
                    }
                elif response.status_code == 404:
                    return None
                else:
                    log.warning("error fetching %s [HTTP %s]", url, response.status_code)
            except:
                log.warning("error fetching %s", url, exc_info=True)
        log.error("failed to fetch info for tenant %s", tenant)
        return None

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]
 def test_account_container_reload(self):
     for server in ("account", "container"):
         ring = Ring("/etc/swift", ring_name=server)
         node = random.choice(ring.get_part_nodes(1))
         self._check_reload(server, node["ip"], node["port"])

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]

#.........这里部分代码省略.........
        """
        In testing, the pool.waitall() call very occasionally failed to return.
        This is an attempt to make sure the replicator finishes its replication
        pass in some eventuality.
        """
        while True:
            eventlet.sleep(self.lockup_timeout)
            if self.replication_count == self.last_replication_count:
                self.logger.error(_("Lockup detected.. killing live coros."))
                self.kill_coros()
            self.last_replication_count = self.replication_count

    def collect_jobs(self):
        """
        Returns a sorted list of jobs (dictionaries) that specify the
        partitions, nodes, etc to be rsynced.
        """
        jobs = []
        ips = whataremyips()
        for local_dev in [dev for dev in self.object_ring.devs
                if dev and dev['ip'] in ips and dev['port'] == self.port]:
            dev_path = join(self.devices_dir, local_dev['device'])
            obj_path = join(dev_path, 'objects')
            tmp_path = join(dev_path, 'tmp')
            if self.mount_check and not os.path.ismount(dev_path):
                self.logger.warn(_('%s is not mounted'), local_dev['device'])
                continue
            unlink_older_than(tmp_path, time.time() - self.reclaim_age)
            if not os.path.exists(obj_path):
                continue
            for partition in os.listdir(obj_path):
                try:
                    part_nodes = \
                        self.object_ring.get_part_nodes(int(partition))
                    nodes = [node for node in part_nodes
                             if node['id'] != local_dev['id']]
                    jobs.append(dict(path=join(obj_path, partition),
                        device=local_dev['device'],
                        nodes=nodes,
                        delete=len(nodes) > len(part_nodes) - 1,
                        partition=partition))
                except ValueError:
                    continue
        random.shuffle(jobs)
        # Partititons that need to be deleted take priority
        jobs.sort(key=lambda job: not job['delete'])
        self.job_count = len(jobs)
        return jobs

    def replicate(self):
        """Run a replication pass"""
        self.start = time.time()
        self.suffix_count = 0
        self.suffix_sync = 0
        self.suffix_hash = 0
        self.replication_count = 0
        self.last_replication_count = -1
        self.partition_times = []
        stats = eventlet.spawn(self.heartbeat)
        lockup_detector = eventlet.spawn(self.detect_lockups)
        eventlet.sleep()  # Give spawns a cycle
        try:
            self.run_pool = GreenPool(size=self.concurrency)
            jobs = self.collect_jobs()
            for job in jobs:
                if not self.check_ring():

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]

#.........这里部分代码省略.........
        """
        Returns a sorted list of jobs (dictionaries) that specify the
        partitions, nodes, etc to be synced.
        """
        jobs = []
        ips = whataremyips()
        for local_dev in [dev for dev in self.object_ring.devs
                          if dev and dev['replication_ip'] in ips and
                          dev['replication_port'] == self.port]:
            dev_path = join(self.devices_dir, local_dev['device'])
            obj_path = join(dev_path, 'objects')
            tmp_path = join(dev_path, 'tmp')
            if self.mount_check and not ismount(dev_path):
                self.logger.warn(_('%s is not mounted'), local_dev['device'])
                continue
            unlink_older_than(tmp_path, time.time() - self.reclaim_age)
            if not os.path.exists(obj_path):
                try:
                    mkdirs(obj_path)
                except Exception:
                    self.logger.exception('ERROR creating %s' % obj_path)
                continue
            for partition in os.listdir(obj_path):
                try:
                    job_path = join(obj_path, partition)
                    if isfile(job_path):
                        # Clean up any (probably zero-byte) files where a
                        # partition should be.
                        self.logger.warning('Removing partition directory '
                                            'which was a file: %s', job_path)
                        os.remove(job_path)
                        continue
                    part_nodes = \
                        self.object_ring.get_part_nodes(int(partition))
		#MODIFIED LightSync
                    for mypos in range(len(part_nodes)):
                        if part_nodes[mypos]['id'] == local_dev['id']:
                            break
                    nodes = part_nodes[mypos+1:]+part_nodes[:mypos]
		##
                    jobs.append(
                        dict(path=job_path,
                             device=local_dev['device'],
                             nodes=nodes,
                             delete=len(nodes) > len(part_nodes) - 1,
                             partition=partition))
                except (ValueError, OSError):
                    continue
        random.shuffle(jobs)
        if self.handoffs_first:
            # Move the handoff parts to the front of the list
            jobs.sort(key=lambda job: not job['delete'])
        self.job_count = len(jobs)
        return jobs

    def replicate(self, override_devices=None, override_partitions=None):
        """Run a replication pass"""
        self.start = time.time()
        self.suffix_count = 0
        self.suffix_sync = 0
        self.suffix_hash = 0
        self.replication_count = 0
        self.last_replication_count = -1
        self.partition_times = []

        if override_devices is None:

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_part_nodes [as 别名]

#.........这里部分代码省略.........

    def collect_jobs(self):
        """
        Returns a sorted list of jobs (dictionaries) that specify the
        partitions, nodes, etc to be rsynced.
        """
        jobs = []
        ips = whataremyips()
        for local_dev in [
            dev for dev in self.object_ring.devs if dev and dev["ip"] in ips and dev["port"] == self.port
        ]:
            dev_path = join(self.devices_dir, local_dev["device"])
            obj_path = join(dev_path, "objects")
            tmp_path = join(dev_path, "tmp")
            if self.mount_check and not os.path.ismount(dev_path):
                self.logger.warn(_("%s is not mounted"), local_dev["device"])
                continue
            unlink_older_than(tmp_path, time.time() - self.reclaim_age)
            if not os.path.exists(obj_path):
                try:
                    mkdirs(obj_path)
                except Exception:
                    self.logger.exception("ERROR creating %s" % obj_path)
                continue
            for partition in os.listdir(obj_path):
                try:
                    job_path = join(obj_path, partition)
                    if isfile(job_path):
                        # Clean up any (probably zero-byte) files where a
                        # partition should be.
                        self.logger.warning("Removing partition directory " "which was a file: %s", job_path)
                        os.remove(job_path)
                        continue
                    part_nodes = self.object_ring.get_part_nodes(int(partition))
                    nodes = [node for node in part_nodes if node["id"] != local_dev["id"]]
                    jobs.append(
                        dict(
                            path=job_path,
                            device=local_dev["device"],
                            nodes=nodes,
                            delete=len(nodes) > len(part_nodes) - 1,
                            partition=partition,
                        )
                    )
                except (ValueError, OSError):
                    continue
        random.shuffle(jobs)
        self.job_count = len(jobs)
        return jobs

    def replicate(self, override_devices=None, override_partitions=None):
        """Run a replication pass"""
        self.start = time.time()
        self.suffix_count = 0
        self.suffix_sync = 0
        self.suffix_hash = 0
        self.replication_count = 0
        self.last_replication_count = -1
        self.partition_times = []

        if override_devices is None:
            override_devices = []
        if override_partitions is None:
            override_partitions = []

        stats = eventlet.spawn(self.heartbeat)