本文整理汇总了Python中swift.obj.diskfile.DiskFileManager.pickle_async_update方法的典型用法代码示例。如果您正苦于以下问题：Python DiskFileManager.pickle_async_update方法的具体用法？Python DiskFileManager.pickle_async_update怎么用？Python DiskFileManager.pickle_async_update使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类swift.obj.diskfile.DiskFileManager的用法示例。
在下文中一共展示了DiskFileManager.pickle_async_update方法的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: do_test
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
def do_test(headers_out, expected):
# write an async
dfmanager = DiskFileManager(conf, daemon.logger)
account, container, obj = 'a', 'c', 'o'
op = 'PUT'
data = {'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out}
dfmanager.pickle_async_update(self.sda1, account, container, obj,
data, next(ts_iter), policies[0])
request_log = []
def capture(*args, **kwargs):
request_log.append((args, kwargs))
# run once
fake_status_codes = [
200, # object update success
200, # object update success
200, # object update conflict
]
with mocked_http_conn(*fake_status_codes, give_connect=capture):
daemon.run_once()
self.assertEqual(len(fake_status_codes), len(request_log))
for request_args, request_kwargs in request_log:
ip, part, method, path, headers, qs, ssl = request_args
self.assertEqual(method, 'PUT')
self.assertDictEqual(expected, headers)
self.assertEqual(
daemon.logger.get_increment_counts(),
{'successes': 1, 'unlinks': 1, 'async_pendings': 1})
self.assertFalse(os.listdir(async_dir))
daemon.logger.clear()示例2: test_obj_put_async_updates
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
def test_obj_put_async_updates(self):
ts = (normalize_timestamp(t) for t in
itertools.count(int(time())))
policy = random.choice(list(POLICIES))
# setup updater
conf = {
'devices': self.devices_dir,
'mount_check': 'false',
'swift_dir': self.testdir,
}
daemon = object_updater.ObjectUpdater(conf, logger=self.logger)
async_dir = os.path.join(self.sda1, get_async_dir(policy))
os.mkdir(async_dir)
# write an async
dfmanager = DiskFileManager(conf, daemon.logger)
account, container, obj = 'a', 'c', 'o'
op = 'PUT'
headers_out = swob.HeaderKeyDict({
'x-size': 0,
'x-content-type': 'text/plain',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-timestamp': next(ts),
'X-Backend-Storage-Policy-Index': int(policy),
})
data = {'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out}
dfmanager.pickle_async_update(self.sda1, account, container, obj,
data, next(ts), policy)
request_log = []
def capture(*args, **kwargs):
request_log.append((args, kwargs))
# run once
fake_status_codes = [
200, # object update success
200, # object update success
200, # object update conflict
]
with mocked_http_conn(*fake_status_codes, give_connect=capture):
daemon.run_once()
self.assertEqual(len(fake_status_codes), len(request_log))
for request_args, request_kwargs in request_log:
ip, part, method, path, headers, qs, ssl = request_args
self.assertEqual(method, 'PUT')
self.assertEqual(headers['X-Backend-Storage-Policy-Index'],
str(int(policy)))
self.assertEqual(daemon.logger.get_increment_counts(),
{'successes': 1, 'unlinks': 1, 'async_pendings': 1})示例3: test_obj_put_legacy_updates
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
def test_obj_put_legacy_updates(self):
ts = (normalize_timestamp(t) for t in itertools.count(int(time())))
policy = POLICIES.get_by_index(0)
# setup updater
conf = {"devices": self.devices_dir, "mount_check": "false", "swift_dir": self.testdir}
async_dir = os.path.join(self.sda1, get_async_dir(policy.idx))
os.mkdir(async_dir)
account, container, obj = "a", "c", "o"
# write an async
for op in ("PUT", "DELETE"):
self.logger._clear()
daemon = object_updater.ObjectUpdater(conf, logger=self.logger)
dfmanager = DiskFileManager(conf, daemon.logger)
# don't include storage-policy-index in headers_out pickle
headers_out = swob.HeaderKeyDict(
{
"x-size": 0,
"x-content-type": "text/plain",
"x-etag": "d41d8cd98f00b204e9800998ecf8427e",
"x-timestamp": ts.next(),
}
)
data = {"op": op, "account": account, "container": container, "obj": obj, "headers": headers_out}
dfmanager.pickle_async_update(self.sda1, account, container, obj, data, ts.next(), policy.idx)
request_log = []
def capture(*args, **kwargs):
request_log.append((args, kwargs))
# run once
fake_status_codes = [200, 200, 200]
with mocked_http_conn(*fake_status_codes, give_connect=capture):
daemon.run_once()
self.assertEqual(len(fake_status_codes), len(request_log))
for request_args, request_kwargs in request_log:
ip, part, method, path, headers, qs, ssl = request_args
self.assertEqual(method, op)
self.assertEqual(headers["X-Backend-Storage-Policy-Index"], str(policy.idx))
self.assertEqual(daemon.logger.get_increment_counts(), {"successes": 1, "unlinks": 1, "async_pendings": 1})示例4: ObjectController
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
#.........这里部分代码省略.........
:param contdevice: device name that the container is on
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.rsplit(':', 1)
conn = http_connect(ip, port, contdevice, partition, op,
full_path, headers_out)
with Timeout(self.node_timeout):
response = conn.getresponse()
response.read()
if is_success(response.status):
return
else:
self.logger.error(_(
'ERROR Container update failed '
'(saving for async update later): %(status)d '
'response from %(ip)s:%(port)s/%(dev)s'),
{'status': response.status, 'ip': ip, 'port': port,
'dev': contdevice})
except (Exception, Timeout):
self.logger.exception(_(
'ERROR container update failed with '
'%(ip)s:%(port)s/%(dev)s (saving for async update later)'),
{'ip': ip, 'port': port, 'dev': contdevice})
data = {'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out}
timestamp = headers_out['x-timestamp']
self._diskfile_mgr.pickle_async_update(objdevice, account, container,
obj, data, timestamp)
def container_update(self, op, account, container, obj, request,
headers_out, objdevice):
"""
Update the container when objects are updated.
:param op: operation performed (ex: 'PUT', or 'DELETE')
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param request: the original request object driving the update
:param headers_out: dictionary of headers to send in the container
request(s)
:param objdevice: device name that the object is in
"""
headers_in = request.headers
conthosts = [h.strip() for h in
headers_in.get('X-Container-Host', '').split(',')]
contdevices = [d.strip() for d in
headers_in.get('X-Container-Device', '').split(',')]
contpartition = headers_in.get('X-Container-Partition', '')
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(_('ERROR Container update failed: different '
'numbers of hosts and devices in request: '
'"%s" vs "%s"') %
(headers_in.get('X-Container-Host', ''),
headers_in.get('X-Container-Device', '')))
return示例5: ObjectController
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
#.........这里部分代码省略.........
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
:param policy_index: the associated storage policy index
"""
headers_out["user-agent"] = "object-server %s" % os.getpid()
full_path = "/%s/%s/%s" % (account, container, obj)
if all([host, partition, contdevice]):
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.rsplit(":", 1)
conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out)
with Timeout(self.node_timeout):
response = conn.getresponse()
response.read()
if is_success(response.status):
return
else:
self.logger.error(
_(
"ERROR Container update failed "
"(saving for async update later): %(status)d "
"response from %(ip)s:%(port)s/%(dev)s"
),
{"status": response.status, "ip": ip, "port": port, "dev": contdevice},
)
except (Exception, Timeout):
self.logger.exception(
_("ERROR container update failed with " "%(ip)s:%(port)s/%(dev)s (saving for async update later)"),
{"ip": ip, "port": port, "dev": contdevice},
)
data = {"op": op, "account": account, "container": container, "obj": obj, "headers": headers_out}
timestamp = headers_out["x-timestamp"]
self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp, policy_index)
def container_update(self, op, account, container, obj, request, headers_out, objdevice, policy_idx):
"""
Update the container when objects are updated.
:param op: operation performed (ex: 'PUT', or 'DELETE')
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param request: the original request object driving the update
:param headers_out: dictionary of headers to send in the container
request(s)
:param objdevice: device name that the object is in
"""
headers_in = request.headers
conthosts = [h.strip() for h in headers_in.get("X-Container-Host", "").split(",")]
contdevices = [d.strip() for d in headers_in.get("X-Container-Device", "").split(",")]
contpartition = headers_in.get("X-Container-Partition", "")
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(
_(
"ERROR Container update failed: different "
"numbers of hosts and devices in request: "
'"%s" vs "%s"'
)
% (headers_in.get("X-Container-Host", ""), headers_in.get("X-Container-Device", ""))
)
return
示例6: ObjectController
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
#.........这里部分代码省略.........
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
:param policy_index: the associated storage policy index
"""
headers_out['user-agent'] = 'object-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.rsplit(':', 1)
conn = http_connect(ip, port, contdevice, partition, op,
full_path, headers_out)
with Timeout(self.node_timeout):
response = conn.getresponse()
response.read()
if is_success(response.status):
return
else:
self.logger.error(_(
'ERROR Container update failed '
'(saving for async update later): %(status)d '
'response from %(ip)s:%(port)s/%(dev)s'),
{'status': response.status, 'ip': ip, 'port': port,
'dev': contdevice})
except (Exception, Timeout):
self.logger.exception(_(
'ERROR container update failed with '
'%(ip)s:%(port)s/%(dev)s (saving for async update later)'),
{'ip': ip, 'port': port, 'dev': contdevice})
data = {'op': op, 'account': account, 'container': container,
'obj': obj, 'headers': headers_out}
timestamp = headers_out['x-timestamp']
self._diskfile_mgr.pickle_async_update(objdevice, account, container,
obj, data, timestamp,
policy_index)
def container_update(self, op, account, container, obj, request,
headers_out, objdevice, policy_idx):
"""
Update the container when objects are updated.
:param op: operation performed (ex: 'PUT', or 'DELETE')
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param request: the original request object driving the update
:param headers_out: dictionary of headers to send in the container
request(s)
:param objdevice: device name that the object is in
"""
headers_in = request.headers
conthosts = [h.strip() for h in
headers_in.get('X-Container-Host', '').split(',')]
contdevices = [d.strip() for d in
headers_in.get('X-Container-Device', '').split(',')]
contpartition = headers_in.get('X-Container-Partition', '')
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(_('ERROR Container update failed: different '
'numbers of hosts and devices in request: '
'"%s" vs "%s"') %
(headers_in.get('X-Container-Host', ''),
headers_in.get('X-Container-Device', '')))示例7: ObjectController
# 需要导入模块: from swift.obj.diskfile import DiskFileManager [as 别名]
# 或者: from swift.obj.diskfile.DiskFileManager import pickle_async_update [as 别名]
#.........这里部分代码省略.........
:param contdevice: device name that the container is on
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
"""
headers_out["user-agent"] = "obj-server %s" % os.getpid()
full_path = "/%s/%s/%s" % (account, container, obj)
if all([host, partition, contdevice]):
try:
with ConnectionTimeout(self.conn_timeout):
ip, port = host.rsplit(":", 1)
conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out)
with Timeout(self.node_timeout):
response = conn.getresponse()
response.read()
if is_success(response.status):
return
else:
self.logger.error(
_(
"ERROR Container update failed "
"(saving for async update later): %(status)d "
"response from %(ip)s:%(port)s/%(dev)s"
),
{"status": response.status, "ip": ip, "port": port, "dev": contdevice},
)
except (Exception, Timeout):
self.logger.exception(
_("ERROR container update failed with " "%(ip)s:%(port)s/%(dev)s (saving for async update later)"),
{"ip": ip, "port": port, "dev": contdevice},
)
data = {"op": op, "account": account, "container": container, "obj": obj, "headers": headers_out}
timestamp = headers_out["x-timestamp"]
self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp)
def container_update(self, op, account, container, obj, request, headers_out, objdevice):
"""
Update the container when objects are updated.
:param op: operation performed (ex: 'PUT', or 'DELETE')
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param request: the original request object driving the update
:param headers_out: dictionary of headers to send in the container
request(s)
:param objdevice: device name that the object is in
"""
headers_in = request.headers
conthosts = [h.strip() for h in headers_in.get("X-Container-Host", "").split(",")]
contdevices = [d.strip() for d in headers_in.get("X-Container-Device", "").split(",")]
contpartition = headers_in.get("X-Container-Partition", "")
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(
_(
"ERROR Container update failed: different "
"numbers of hosts and devices in request: "
'"%s" vs "%s"'
)
% (headers_in.get("X-Container-Host", ""), headers_in.get("X-Container-Device", ""))
)
return