Python utils.rsync_ip函数代码示例

    def _rsync_db(self, broker, device, http, local_id,
            replicate_method='complete_rsync', replicate_timeout=None):
        """
        Sync a whole db using rsync.

        :param broker: DB broker object of DB to be synced
        :param device: device to sync to
        :param http: ReplConnection object
        :param local_id: unique ID of the local database replica
        :param replicate_method: remote operation to perform after rsync
        :param replicate_timeout: timeout to wait in seconds
        """
        device_ip = rsync_ip(device['ip'])
        if self.vm_test_mode:
            remote_file = '%s::%s%s/%s/tmp/%s' % (device_ip,
                    self.server_type, device['port'], device['device'],
                    local_id)
        else:
            remote_file = '%s::%s/%s/tmp/%s' % (device_ip,
                    self.server_type, device['device'], local_id)
        mtime = os.path.getmtime(broker.db_file)
        if not self._rsync_file(broker.db_file, remote_file):
            return False
        # perform block-level sync if the db was modified during the first sync
        if os.path.exists(broker.db_file + '-journal') or \
                    os.path.getmtime(broker.db_file) > mtime:
            # grab a lock so nobody else can modify it
            with broker.lock():
                if not self._rsync_file(broker.db_file, remote_file, False):
                    return False
        with Timeout(replicate_timeout or self.node_timeout):
            response = http.replicate(replicate_method, local_id)
        return response and response.status >= 200 and response.status < 300

 def rsync(self, node, job, suffixes):
     """
     Uses rsync to implement the sync method. This was the first
     sync method in Swift.
     """
     if not os.path.exists(job['path']):
         return False
     args = [
         'rsync',
         '--recursive',
         '--whole-file',
         '--human-readable',
         '--xattrs',
         '--itemize-changes',
         '--ignore-existing',
         '--timeout=%s' % self.rsync_io_timeout,
         '--contimeout=%s' % self.rsync_io_timeout,
         '--bwlimit=%s' % self.rsync_bwlimit,
     ]
     node_ip = rsync_ip(node['replication_ip'])
     if self.vm_test_mode:
         rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
     else:
         rsync_module = '%s::object' % node_ip
     had_any = False
     for suffix in suffixes:
         spath = join(job['path'], suffix)
         if os.path.exists(spath):
             args.append(spath)
             had_any = True
     if not had_any:
         return False
     args.append(join(rsync_module, node['device'],
                 'objects', job['partition']))
     return self._rsync(args) == 0

 def rsync(self, node, job, suffixes):
     """
     Uses rsync to implement the sync method. This was the first
     sync method in Swift.
     """
     if not os.path.exists(job["path"]):
         return False
     args = [
         "rsync",
         "--recursive",
         "--whole-file",
         "--human-readable",
         "--xattrs",
         "--itemize-changes",
         "--ignore-existing",
         "--timeout=%s" % self.rsync_io_timeout,
         "--contimeout=%s" % self.rsync_io_timeout,
         "--bwlimit=%s" % self.rsync_bwlimit,
     ]
     node_ip = rsync_ip(node["replication_ip"])
     if self.vm_test_mode:
         rsync_module = "%s::object%s" % (node_ip, node["replication_port"])
     else:
         rsync_module = "%s::object" % node_ip
     had_any = False
     for suffix in suffixes:
         spath = join(job["path"], suffix)
         if os.path.exists(spath):
             args.append(spath)
             had_any = True
     if not had_any:
         return False
     data_dir = get_data_dir(job["policy_idx"])
     args.append(join(rsync_module, node["device"], data_dir, job["partition"]))
     return self._rsync(args) == 0

    def rsync(self, job):
        """
        Uses rsync to implement the sync method. This was the first
        sync method in Swift.
        """

        if not os.path.exists(job['path']):
            if self.test:
                print "Error: the path %s does not exists" % job['path']
            return False, {}

        args = [
            'rsync',
            '-a',
            '--whole-file',
            '--human-readable',
            '--xattrs',
            '--ignore-existing',
        ]

        node = job['node']
        node_ip = rsync_ip(node['replication_ip'])
        rsync_module = '%s:%s' % (node_ip, job['remote_path'])

        args.append(job['path'])
        args.append(rsync_module)

        if not self.test:
            return self._rsync(args) == 0, {}
        else:
            print " ".join(args)
            return True, {}

    def create_remote_directory(self, job):
        """
        Creates a temporal directory, at remote server.

        :param job: information about the partition being synced

        """
        node = job['node']

        args = ["ssh", rsync_ip(node['replication_ip']),
                "mkdir", "-p", job['remote_path']]

        if not self.test:
            proc = subprocess.Popen(args,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)

            results = proc.stdout.read()
            ret_val = proc.wait()

            #TODO: ret_val check
            (results, ret_val)

        else:
            print " ".join(args)

def rsync(partition, device, suffix):
        node_ip = rsync_ip('127.0.0.1')
	port=''
	if device == 'd2':
		port = '6020'
	if device == 'd3':
		port = '6030'
        rsync_module = '%s::object%s' %(node_ip,port)
        spath = join('/srv/node/ssd/objects/%s' %(partition),suffix[1])
	print(suffix)
        args = [
            'rsync',
            '--recursive',
            '--whole-file',
            '--human-readable',
            '--xattrs',
            '--itemize-changes',
            '--ignore-existing',
            '--timeout=30',
            '--contimeout=30',
        ]
        args.append(spath)
        args.append(join(rsync_module, device, 'objects', partition))
	start_time = time.time()
        ret_val = None
        try:
                with Timeout(900):
                    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    results = proc.stdout.read()
                    ret_val = proc.wait()

        except Timeout:
            print("Killing long-running rsync: %s", str(args))
            proc.kill()
            return 1
	total_time = time.time() - start_time
        for result in results.split('\n'):
            if result == '':
                continue
            if result.startswith('cd+'):
                continue
	    if not ret_val:
                print(result)
            else:
                print(result)
        if ret_val:
            print('Bad rsync return code: %(args)s -> %(ret)d',
                              {'args': str(args), 'ret': ret_val})
        elif results:
            print("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
                {'src': args[-2], 'dst': args[-1], 'time': total_time})
        else:
            print("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
                {'src': args[-2], 'dst': args[-1], 'time': total_time})
        return ret_val

def rsync(partition, device):
	node_ip = rsync_ip(device['ip'])
	rsync_module = '[email protected]%s:' %(node_ip)
	spath = '/SSD/'+str(partition)
	args = [
            'rsync',
            '--recursive',
            '--whole-file',
            '--human-readable',
            '--xattrs',
            '--itemize-changes',
            '--ignore-existing',
            '--timeout=30',
            '--archive'
    ]
	args.append(spath)
	# args.append(join(rsync_module,'srv/node',device['device'],'objects',partition))
	args.append(join(rsync_module,'srv/node',device['device'],'objects/'))
	logging.info("===args===%s",str(args))
	start_time = time.time()
	ret_val = None
	try:
		with Timeout(900):
			proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
			results = proc.stdout.read()
			ret_val = proc.wait()

	except Timeout:
		logging.info("Killing long-running rsync: %s", str(args))
		proc.kill()
		return 1
	total_time = time.time() - start_time
	logging.info("===Total time===%s",str(total_time))
	for result in results.split('\n'):
	    if result == '':
	        continue
	    if result.startswith('cd+'):
	        continue
	if not ret_val:
		logging.info(result)
	else:
		logging.info(result)
	if ret_val:
		logging.info('Bad rsync return code: %(args)s -> %(ret)d',
	                      {'args': str(args), 'ret': ret_val})
	elif results:
		logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
	        {'src': args[-2], 'dst': args[-1], 'time': total_time})
	else:
		logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
	        {'src': args[-2], 'dst': args[-1], 'time': total_time})

    def rsync(self, node, job, suffixes):
        """
	从一个远程结点上的partition同步一个文件
        Synchronize local suffix directories from a partition with a remote
        node.

        :param node: the "dev" entry for the remote node to sync with
        :param job: information about the partition being synced
        :param suffixes: a list of suffixes which need to be pushed

        :returns: boolean indicating success or failure
        """
        if not os.path.exists(job['path']):
            return False
        args = [
            'rsync',
            '--recursive',
            '--whole-file',
            '--human-readable',
            '--xattrs',
            '--itemize-changes',
            '--ignore-existing',
            '--timeout=%s' % self.rsync_io_timeout,
            '--contimeout=%s' % self.rsync_io_timeout,
            '--bwlimit=%s' % self.rsync_bwlimit,
        ]
        node_ip = rsync_ip(node['replication_ip'])
        if self.vm_test_mode:
            rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
        else:
            rsync_module = '%s::object' % node_ip
        had_any = False
        for suffix in suffixes:
            spath = join(job['path'], suffix)
            if os.path.exists(spath):
                args.append(spath)
                had_any = True
        if not had_any:
            return False
        args.append(join(rsync_module, node['device'],
                    'objects', job['partition']))
        return self._rsync(args) == 0

 def rsync(self, node, job, suffixes):
     """
     Uses rsync to implement the sync method. This was the first
     sync method in Swift.
     """
     if not os.path.exists(job['path']):
         return False, {}
     args = [
         'rsync',
         '--recursive',
         '--whole-file',
         '--human-readable',
         '--xattrs',
         '--itemize-changes',
         '--ignore-existing',
         '--timeout=%s' % self.rsync_io_timeout,
         '--contimeout=%s' % self.rsync_io_timeout,
         '--bwlimit=%s' % self.rsync_bwlimit,
     ]
     if self.rsync_compress and \
             job['region'] != node['region']:
         # Allow for compression, but only if the remote node is in
         # a different region than the local one.
         args.append('--compress')
     node_ip = rsync_ip(node['replication_ip'])
     if self.vm_test_mode:
         rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
     else:
         rsync_module = '%s::object' % node_ip
     had_any = False
     for suffix in suffixes:
         spath = join(job['path'], suffix)
         if os.path.exists(spath):
             args.append(spath)
             had_any = True
     if not had_any:
         return False, {}
     data_dir = get_data_dir(job['policy'])
     args.append(join(rsync_module, node['device'],
                 data_dir, job['partition']))
     return self._rsync(args) == 0, {}

 def rsync(self, node, job, suffixes):
     """
     Uses rsync to implement the sync method. This was the first
     sync method in Swift.
     """
     curframe = inspect.currentframe()
     calframe = inspect.getouterframes(curframe, 2)
     f = open("/home/hduser/log.txt","w")
     f.write(str(calframe))
     f.close()
     if not os.path.exists(job['path']):
         return False, set()
     args = [
         'rsync',
         '--recursive',
         '--whole-file',
         '--human-readable',
         '--xattrs',
         '--itemize-changes',
         '--ignore-existing',
         '--timeout=%s' % self.rsync_io_timeout,
         '--contimeout=%s' % self.rsync_io_timeout,
         '--bwlimit=%s' % self.rsync_bwlimit,
     ]
     node_ip = rsync_ip(node['replication_ip'])
     if self.vm_test_mode:
         rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
     else:
         rsync_module = '%s::object' % node_ip
     had_any = False
     for suffix in suffixes:
         spath = join(job['path'], suffix)
         if os.path.exists(spath):
             args.append(spath)
             had_any = True
     if not had_any:
         return False, set()
     data_dir = get_data_dir(job['policy_idx'])
     args.append(join(rsync_module, node['device'],
                 data_dir, job['partition']))
     return self._rsync(args) == 0, set()

    def rsync(self, node, job, suffixes):
        """
        Synchronize local suffix directories from a partition with a remote
        node.

        :param node: the "dev" entry for the remote node to sync with
        :param job: information about the partition being synced
        :param suffixes: a list of suffixes which need to be pushed

        :returns: boolean indicating success or failure
        """
        if not os.path.exists(job["path"]):
            return False
        args = [
            "rsync",
            "--recursive",
            "--whole-file",
            "--human-readable",
            "--xattrs",
            "--itemize-changes",
            "--ignore-existing",
            "--timeout=%s" % self.rsync_io_timeout,
            "--contimeout=%s" % self.rsync_io_timeout,
        ]
        node_ip = rsync_ip(node["ip"])
        if self.vm_test_mode:
            rsync_module = "%s::object%s" % (node_ip, node["port"])
        else:
            rsync_module = "%s::object" % node_ip
        had_any = False
        for suffix in suffixes:
            spath = join(job["path"], suffix)
            if os.path.exists(spath):
                args.append(spath)
                had_any = True
        if not had_any:
            return False
        args.append(join(rsync_module, node["device"], "objects", job["partition"]))
        return self._rsync(args) == 0

 def test_rsync_ip_ipv6_ipv4_compatible(self):
     self.assertEqual(
         utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')

 def test_rsync_ip_ipv6_random_ip(self):
     self.assertEqual(
         utils.rsync_ip('fe80:0000:0000:0000:0202:b3ff:fe1e:8329'),
         '[fe80:0000:0000:0000:0202:b3ff:fe1e:8329]')

 def test_rsync_ip_ipv4_localhost(self):
     self.assertEqual(utils.rsync_ip('127.0.0.1'), '127.0.0.1')