本文整理匯總了Python中pySmartDL.SmartDL方法的典型用法代碼示例。如果您正苦於以下問題:Python pySmartDL.SmartDL方法的具體用法?Python pySmartDL.SmartDL怎麽用?Python pySmartDL.SmartDL使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類pySmartDL
的用法示例。
在下文中一共展示了pySmartDL.SmartDL方法的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: download
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def download(url, filename, cache_directory):
filename_cache = url.split('/')[-1]
filename_cache = ''.join([c for c in filename_cache if c.isdigit() or c.isalpha()])
filename_cache = cache_directory + "/" + filename_cache
if os.path.exists(filename):
return
elif os.path.exists(filename_cache):
print("Already downloaded")
shutil.copyfile(filename_cache, filename)
else:
print("\nDownloading {} from {}".format(filename, url))
os.mkdir(cache_directory)
# wget.download(url, out=filename_cache)
obj = SmartDL(url, filename_cache)
obj.start()
shutil.copyfile(filename_cache, filename)
示例2: _download
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def _download(self, requirement, dest):
url = self._get_url(requirement)
wheel_name = url.split("/")[-1]
print("Downloading package . . .")
print(url)
print(wheel_name)
if dest is not None:
# Ensure the download directory
if not exists(dest):
os.makedirs(dest)
else:
dest = self._get_pipwin_dir()
wheel_file = join(dest, wheel_name)
if exists(wheel_file):
print("File " + wheel_file + " already exists")
return wheel_file
obj = SmartDL(url, dest)
obj.start()
return wheel_file
示例3: geckodown
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def geckodown(directory):
source=requests.get("https://github.com/mozilla/geckodriver/releases/latest").text
soup=BeautifulSoup(source.encode("utf-8"),'lxml')
vr=str(soup.title.text.encode("utf-8")).split(' ')[1]
container="https://github.com/mozilla/geckodriver/releases/download/"+vr+"/geckodriver-"+vr+'-'+comb
print("Downloading from: "+str(container))
try:
url = container
dest = directory
obj = SmartDL(url, dest)
obj.start()
path=obj.get_dest()
print(os.path.join(directory,'geckodriver-'+vr+'-linux64.zip'))
filepath=os.path.join(directory,'geckodriver-'+vr+'-'+comb)
if (filepath.endswith("tar.gz")):
tar = tarfile.open(filepath,'r:*')
tar.extractall(directory)
tar.close()
#print "Extracted in Current Directory"
print("Use selenium driver path as "+os.path.join(directory,"geckodriver"))
except Exception as e:
print('Issues updating with error '+str(e))
示例4: geckodown
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def geckodown(directory):
source=requests.get("https://github.com/mozilla/geckodriver/releases/latest").text
soup=BeautifulSoup(source.encode("utf-8"),'lxml')
vr=str(soup.title.text.encode("utf-8")).split(' ')[1]
container="https://github.com/mozilla/geckodriver/releases/download/"+vr+"/geckodriver-"+vr+'-'+comb
print("Downloading from: "+str(container))
try:
url = container
dest = directory
obj = SmartDL(url, dest)
obj.start()
path=obj.get_dest()
print(os.path.join(directory,'geckodriver-'+vr+'-win64.zip'))
archive=zipfile.ZipFile(os.path.join(directory,'geckodriver-'+vr+'-'+comb))
for files in archive.namelist():
archive.extractall(directory)
print("Use selenium driver path as "+str(directory))
except Exception as e:
print('Issues updating with error '+str(e))
示例5: downonly
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def downonly(redirect_url,local_path,ext,items,ulength):
global i
result=SESSION.get(redirect_url)
if not os.path.exists(local_path) and result.status_code==200:
if ext is not None:
if local_path.endswith(ext):
print(str(ulength-i)+" remaining ==> Downloading: " + str(local_path))
i=i+1
obj = SmartDL(redirect_url, local_path)
obj.start()
path = obj.get_dest()
elif ext is None:
print(str(ulength-i)+" remaining ==> Downloading: " + str(local_path))
i=i+1
obj = SmartDL(redirect_url, local_path)
obj.start()
path = obj.get_dest()
elif result.status_code==429:
raise Exception("rate limit error")
else:
if int(result.status_code)!=200:
print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
elif int(result.status_code)==200:
print("Checking "+str(ulength-i)+" remaining ==> "+"File already exists SKIPPING: "+str(os.path.split(local_path)[-1]))
i=i+1
示例6: test_download
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_download(self):
obj = pySmartDL.SmartDL(self.res_7za920_mirrors, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
obj.start()
self.assertEqual(obj.get_progress_bar(), '[##################]')
data = obj.get_data(binary=True, bytes=2)
self.assertEqual(data, b'PK')
# attempt to start a completed task
with self.assertRaises(RuntimeError) as ctx:
obj.start()
示例7: test_mirrors
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_mirrors(self):
urls = ["http://totally_fake_website/7za.zip", "https://github.com/iTaybb/pySmartDL/raw/master/test/7za920.zip"]
obj = pySmartDL.SmartDL(urls, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
obj.start()
self.assertTrue(obj.isSuccessful())
示例8: test_hash
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_hash(self):
obj = pySmartDL.SmartDL(self.res_7za920_mirrors, progress_bar=False, connect_default_logger=self.enable_logging)
obj.add_hash_verification('sha256' , self.res_7za920_hash) # good hash
obj.start(blocking=False) # no exceptions
obj.wait()
self.assertTrue(obj.isSuccessful())
obj = pySmartDL.SmartDL(self.res_7za920_mirrors, progress_bar=False, connect_default_logger=self.enable_logging)
obj.add_hash_verification('sha256' ,'a'*64) # bad hash
obj.start(blocking=False) # no exceptions
obj.wait()
self.assertFalse(obj.isSuccessful())
self.assertTrue(any([isinstance(e, pySmartDL.HashFailedException) for e in obj.get_errors()]))
示例9: test_pause_unpause
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_pause_unpause(self, testfile=None):
obj = pySmartDL.SmartDL(testfile if testfile else self.res_7za920_mirrors, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
obj.start(blocking=False)
while not obj.get_dl_size():
time.sleep(0.1)
# pause
obj.pause()
time.sleep(0.5)
if obj.get_status() == "finished":
# too bad, the file was too small and was downloaded complectely until we stopped it.
# We should download a bigger file
if self.res_testfile_100mb == testfile:
self.fail("The download got completed before we could stop it, even though we've used a big file. Are we on a 100GB/s internet connection or somethin'?")
return self.test_pause_unpause(testfile=self.res_testfile_100mb)
dl_size = obj.get_dl_size()
# verify download has really stopped
time.sleep(2.5)
self.assertEqual(dl_size, obj.get_dl_size())
# continue
obj.unpause()
time.sleep(2.5)
self.assertNotEqual(dl_size, obj.get_dl_size())
obj.wait()
self.assertTrue(obj.isSuccessful())
示例10: test_stop
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_stop(self):
obj = pySmartDL.SmartDL(self.res_testfile_100mb, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
obj.start(blocking=False)
while not obj.get_dl_size():
time.sleep(0.1)
obj.stop()
obj.wait()
self.assertFalse(obj.isSuccessful())
示例11: test_basic_auth
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_basic_auth(self):
basic_auth_test_url = "https://httpbin.org/basic-auth/user/passwd"
obj = pySmartDL.SmartDL(basic_auth_test_url, progress_bar=False, connect_default_logger=self.enable_logging)
obj.add_basic_authentication('user', 'passwd')
obj.start()
data = obj.get_data()
self.assertTrue(json.loads(data)['authenticated'])
示例12: test_unicode
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_unicode(self):
url = "https://he.wikipedia.org/wiki/ג'חנון"
obj = pySmartDL.SmartDL(url, progress_bar=False, connect_default_logger=self.enable_logging)
obj.start()
示例13: test_timeout
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_timeout(self):
self.assertRaises(socket.timeout, pySmartDL.SmartDL, "https://httpbin.org/delay/10", progress_bar=False, timeout=3, connect_default_logger=self.enable_logging)
obj = pySmartDL.SmartDL("https://httpbin.org/delay/3", progress_bar=False, timeout=15, connect_default_logger=self.enable_logging)
obj.start(blocking=False)
obj.wait()
self.assertTrue(obj.isSuccessful())
示例14: test_custom_headers
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def test_custom_headers(self):
# sending custom user agent
ua = "pySmartDL/1.3.2"
request_args = {"headers": {"User-Agent": ua}}
obj = pySmartDL.SmartDL("http://httpbin.org/headers", request_args=request_args, progress_bar=False)
obj.start()
data = obj.get_json()
self.assertTrue(data['headers']['User-Agent'] == ua)
# passing empty request_args
obj = pySmartDL.SmartDL("http://httpbin.org/headers", request_args={}, progress_bar=False)
obj.start()
self.assertTrue(obj.isSuccessful())
示例15: download_to_local
# 需要導入模塊: import pySmartDL [as 別名]
# 或者: from pySmartDL import SmartDL [as 別名]
def download_to_local(config: Config, local: LocalStorage, sha256: Optional[str] = None,
source_url: Optional[str] = None, path: Optional[str] = None):
if sha256 is not None:
local_path = local.hash_to_file(sha256)
source_url = config.source_url(sha256=sha256)
if source_url is None:
raise RuntimeError("Cannot find source_url for file with hash `%s`. "
"See `lazydata add-source` command." % sha256)
if path is None:
path = config.path(sha256=sha256)
if path is None:
raise RuntimeError("Cannot find path for downloading a file.")
elif source_url is not None:
if path is None:
path = config.path(source_url=source_url)
if path is None:
raise RuntimeError("Cannot find path for downloading a file.")
local_path = Path(path)
else:
raise RuntimeError("Cannot download a file without sha256 and source_url specified.")
local_path.parent.mkdir(parents=True, exist_ok=True)
f = SmartDL(urls=source_url, dest=str(local_path), progress_bar=False)
print("Downloading `%s`" % path)
f.start()
# make sure the sha256 of the just downloaded file is correct
downloaded_sha256 = calculate_file_sha256(str(local_path))
if sha256 is not None and sha256 != downloaded_sha256:
raise RuntimeError("Hash for the downloaded file `%s` is incorrect. "
"File might be corrupted in the remote storage backend." % str(local_path))
local.store_file(path=path)