Python pySmartDL.SmartDL方法代码示例

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def download(url, filename, cache_directory):
    filename_cache = url.split('/')[-1]
    filename_cache = ''.join([c for c in filename_cache if c.isdigit() or c.isalpha()])
    filename_cache = cache_directory + "/" + filename_cache
    if os.path.exists(filename):
        return
    elif os.path.exists(filename_cache):
        print("Already downloaded")
        shutil.copyfile(filename_cache, filename)
    else:
        print("\nDownloading {} from {}".format(filename, url))
        os.mkdir(cache_directory)
        # wget.download(url, out=filename_cache)
        obj = SmartDL(url, filename_cache)
        obj.start()
        shutil.copyfile(filename_cache, filename) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def _download(self, requirement, dest):
        url = self._get_url(requirement)
        wheel_name = url.split("/")[-1]
        print("Downloading package . . .")
        print(url)
        print(wheel_name)

        if dest is not None:
            # Ensure the download directory
            if not exists(dest):
                os.makedirs(dest)
        else:
            dest = self._get_pipwin_dir()

        wheel_file = join(dest, wheel_name)

        if exists(wheel_file):
            print("File " + wheel_file + " already exists")
            return wheel_file

        obj = SmartDL(url, dest)
        obj.start()
        return wheel_file 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def geckodown(directory):
    source=requests.get("https://github.com/mozilla/geckodriver/releases/latest").text
    soup=BeautifulSoup(source.encode("utf-8"),'lxml')
    vr=str(soup.title.text.encode("utf-8")).split(' ')[1]
    container="https://github.com/mozilla/geckodriver/releases/download/"+vr+"/geckodriver-"+vr+'-'+comb
    print("Downloading from: "+str(container))
    try:
        url = container
        dest = directory
        obj = SmartDL(url, dest)
        obj.start()
        path=obj.get_dest()
        print(os.path.join(directory,'geckodriver-'+vr+'-linux64.zip'))
        filepath=os.path.join(directory,'geckodriver-'+vr+'-'+comb)
        if (filepath.endswith("tar.gz")):
            tar = tarfile.open(filepath,'r:*')
            tar.extractall(directory)
            tar.close()
            #print "Extracted in Current Directory"
            print("Use selenium driver path as "+os.path.join(directory,"geckodriver"))
    except Exception as e:
        print('Issues updating with error '+str(e)) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def geckodown(directory):
    source=requests.get("https://github.com/mozilla/geckodriver/releases/latest").text
    soup=BeautifulSoup(source.encode("utf-8"),'lxml')
    vr=str(soup.title.text.encode("utf-8")).split(' ')[1]
    container="https://github.com/mozilla/geckodriver/releases/download/"+vr+"/geckodriver-"+vr+'-'+comb
    print("Downloading from: "+str(container))
    try:
        url = container
        dest = directory
        obj = SmartDL(url, dest)
        obj.start()
        path=obj.get_dest()
        print(os.path.join(directory,'geckodriver-'+vr+'-win64.zip'))
        archive=zipfile.ZipFile(os.path.join(directory,'geckodriver-'+vr+'-'+comb))
        for files in archive.namelist():
            archive.extractall(directory)
        print("Use selenium driver path as "+str(directory))
    except Exception as e:
        print('Issues updating with error '+str(e)) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def downonly(redirect_url,local_path,ext,items,ulength):
    global i
    result=SESSION.get(redirect_url)
    if not os.path.exists(local_path) and result.status_code==200:
        if ext is not None:
            if local_path.endswith(ext):
                print(str(ulength-i)+" remaining ==> Downloading: " + str(local_path))
                i=i+1
                obj = SmartDL(redirect_url, local_path)
                obj.start()
                path = obj.get_dest()
        elif ext is None:
            print(str(ulength-i)+" remaining ==> Downloading: " + str(local_path))
            i=i+1
            obj = SmartDL(redirect_url, local_path)
            obj.start()
            path = obj.get_dest()
    elif result.status_code==429:
        raise Exception("rate limit error")
    else:
        if int(result.status_code)!=200:
            print("Encountered error with code: " + str(result.status_code)+' for '+str(os.path.split(items['name'])[-1]))
        elif int(result.status_code)==200:
            print("Checking "+str(ulength-i)+" remaining ==> "+"File already exists SKIPPING: "+str(os.path.split(local_path)[-1]))
            i=i+1 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_download(self):
        obj = pySmartDL.SmartDL(self.res_7za920_mirrors, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.start()
        self.assertEqual(obj.get_progress_bar(), '[##################]')

        data = obj.get_data(binary=True, bytes=2)
        
        self.assertEqual(data, b'PK')

        # attempt to start a completed task
        with self.assertRaises(RuntimeError) as ctx:
            obj.start() 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_mirrors(self):
        urls = ["http://totally_fake_website/7za.zip", "https://github.com/iTaybb/pySmartDL/raw/master/test/7za920.zip"]
        obj = pySmartDL.SmartDL(urls, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.start()
        
        self.assertTrue(obj.isSuccessful()) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_hash(self):
        obj = pySmartDL.SmartDL(self.res_7za920_mirrors, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.add_hash_verification('sha256' , self.res_7za920_hash)  # good hash
        obj.start(blocking=False)  # no exceptions
        obj.wait()
        
        self.assertTrue(obj.isSuccessful())
        
        obj = pySmartDL.SmartDL(self.res_7za920_mirrors, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.add_hash_verification('sha256' ,'a'*64)  # bad hash
        obj.start(blocking=False)  # no exceptions
        obj.wait()
        
        self.assertFalse(obj.isSuccessful())
        self.assertTrue(any([isinstance(e, pySmartDL.HashFailedException) for e in obj.get_errors()])) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_pause_unpause(self, testfile=None):
        obj = pySmartDL.SmartDL(testfile if testfile else self.res_7za920_mirrors, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.start(blocking=False)
        
        while not obj.get_dl_size():
            time.sleep(0.1)
        
        # pause
        obj.pause()
        time.sleep(0.5)
        if obj.get_status() == "finished":
            # too bad, the file was too small and was downloaded complectely until we stopped it.
            # We should download a bigger file
            if self.res_testfile_100mb == testfile:
                self.fail("The download got completed before we could stop it, even though we've used a big file. Are we on a 100GB/s internet connection or somethin'?")
            return self.test_pause_unpause(testfile=self.res_testfile_100mb)
        
        dl_size = obj.get_dl_size()

        # verify download has really stopped
        time.sleep(2.5)
        self.assertEqual(dl_size, obj.get_dl_size())
        
        # continue
        obj.unpause()
        time.sleep(2.5)
        self.assertNotEqual(dl_size, obj.get_dl_size())
        
        obj.wait()
        self.assertTrue(obj.isSuccessful()) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_stop(self):
        obj = pySmartDL.SmartDL(self.res_testfile_100mb, dest=self.dl_dir, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.start(blocking=False)

        while not obj.get_dl_size():
            time.sleep(0.1)

        obj.stop()
        obj.wait()
        self.assertFalse(obj.isSuccessful()) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_basic_auth(self):
        basic_auth_test_url = "https://httpbin.org/basic-auth/user/passwd"
        obj = pySmartDL.SmartDL(basic_auth_test_url, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.add_basic_authentication('user', 'passwd')
        obj.start()
        data = obj.get_data()
        self.assertTrue(json.loads(data)['authenticated']) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_unicode(self):
        url = "https://he.wikipedia.org/wiki/ג'חנון"
        obj = pySmartDL.SmartDL(url, progress_bar=False, connect_default_logger=self.enable_logging)
        obj.start() 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_timeout(self):
        self.assertRaises(socket.timeout, pySmartDL.SmartDL, "https://httpbin.org/delay/10", progress_bar=False, timeout=3, connect_default_logger=self.enable_logging)

        obj = pySmartDL.SmartDL("https://httpbin.org/delay/3", progress_bar=False, timeout=15, connect_default_logger=self.enable_logging)
        obj.start(blocking=False)
        obj.wait()
        self.assertTrue(obj.isSuccessful()) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def test_custom_headers(self):
        # sending custom user agent
        ua = "pySmartDL/1.3.2"
       	request_args = {"headers": {"User-Agent": ua}}
        obj = pySmartDL.SmartDL("http://httpbin.org/headers", request_args=request_args, progress_bar=False)
        obj.start()
        data = obj.get_json()
        self.assertTrue(data['headers']['User-Agent'] == ua)

        # passing empty request_args
        obj = pySmartDL.SmartDL("http://httpbin.org/headers", request_args={}, progress_bar=False)
        obj.start()
        self.assertTrue(obj.isSuccessful()) 

# 需要导入模块: import pySmartDL [as 别名]
# 或者: from pySmartDL import SmartDL [as 别名]
def download_to_local(config: Config, local: LocalStorage, sha256: Optional[str] = None,
                          source_url: Optional[str] = None, path: Optional[str] = None):
        if sha256 is not None:
            local_path = local.hash_to_file(sha256)
            source_url = config.source_url(sha256=sha256)
            if source_url is None:
                raise RuntimeError("Cannot find source_url for file with hash `%s`. "
                                   "See `lazydata add-source` command." % sha256)
            if path is None:
                path = config.path(sha256=sha256)
                if path is None:
                    raise RuntimeError("Cannot find path for downloading a file.")
        elif source_url is not None:
            if path is None:
                path = config.path(source_url=source_url)
                if path is None:
                    raise RuntimeError("Cannot find path for downloading a file.")
            local_path = Path(path)
        else:
            raise RuntimeError("Cannot download a file without sha256 and source_url specified.")

        local_path.parent.mkdir(parents=True, exist_ok=True)

        f = SmartDL(urls=source_url, dest=str(local_path), progress_bar=False)
        print("Downloading `%s`" % path)
        f.start()
        # make sure the sha256 of the just downloaded file is correct
        downloaded_sha256 = calculate_file_sha256(str(local_path))
        if sha256 is not None and sha256 != downloaded_sha256:
            raise RuntimeError("Hash for the downloaded file `%s` is incorrect. "
                               "File might be corrupted in the remote storage backend." % str(local_path))
        local.store_file(path=path)