本文整理汇总了Python中dipper.models.Dataset.Dataset.set_date_issued方法的典型用法代码示例。如果您正苦于以下问题：Python Dataset.set_date_issued方法的具体用法？Python Dataset.set_date_issued怎么用？Python Dataset.set_date_issued使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dipper.models.Dataset.Dataset的用法示例。
在下文中一共展示了Dataset.set_date_issued方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: fetch
# 需要导入模块: from dipper.models.Dataset import Dataset [as 别名]
# 或者: from dipper.models.Dataset.Dataset import set_date_issued [as 别名]
#.........这里部分代码省略.........
:param is_dl_forced - boolean
:param files dict - override instance files dict
:return: None
"""
fstat = None
if files is None:
files = self.files
for fname in files:
headers = None
filesource = files[fname]
if 'headers' in filesource:
headers = filesource['headers']
LOG.info("Getting %s", fname)
# if the key 'clean' exists in the sources `files` dict
# expose that instead of the longer url
if 'clean' in filesource and filesource['clean'] is not None:
self.dataset.setFileAccessUrl(filesource['clean'])
else:
self.dataset.setFileAccessUrl(filesource['url'])
LOG.info('Fetching %s', filesource['url'])
self.fetch_from_url(
filesource['url'], '/'.join((self.rawdir, filesource['file'])),
is_dl_forced, headers)
fstat = os.stat('/'.join((self.rawdir, filesource['file'])))
# only keeping the date from the last file
filedate = datetime.utcfromtimestamp(fstat[ST_CTIME]).strftime("%Y-%m-%d")
# FIXME
# change this so the date is attached only to each file, not the entire dataset
self.dataset.set_date_issued(filedate)
def fetch_from_url(
self, remotefile, localfile=None, is_dl_forced=False, headers=None):
"""
Given a remote url and a local filename, attempt to determine
if the remote file is newer; if it is,
fetch the remote file and save it to the specified localfile,
reporting the basic file information once it is downloaded
:param remotefile: URL of remote file to fetch
:param localfile: pathname of file to save locally
:return: None
"""
response = None
if ((is_dl_forced is True) or localfile is None or
(self.checkIfRemoteIsNewer(remotefile, localfile, headers))):
# TODO url verification, etc
if headers is None:
headers = self._get_default_request_headers()
request = urllib.request.Request(remotefile, headers=headers)
response = urllib.request.urlopen(request)
if localfile is not None:
with open(localfile, 'wb') as binwrite:
while True:
chunk = response.read(CHUNK)
if not chunk:
break
binwrite.write(chunk)