Python util.download函数代码示例

def download_images(course, threads_folder, thread):
    """
    Download images in given thread.
    The given thread object will be mutated.
    """
    posts = thread['posts']
    comments = thread['comments']

    thread_id = thread['id']
    thread_page = thread['start_page']

    images = []
    last_post_is_full = False
    for post in reversed(posts):
        if 'post_text' in post:
            text = post['post_text']
            text = find_images(text, images, thread_id, thread_page)
            post['post_text'] = text
            last_post_is_full = True
        elif last_post_is_full:
            break

    for comment in comments:
        text = comment['comment_text']
        text = find_images(text, images, thread_id, thread_page)
        comment['comment_text'] = text

    for url, path in images:
        path = '{}/{}'.format(threads_folder, path)
        util.download(url, path, course.get_cookie_file(), resume=True)

def download_thread(course, threads_folder, thread_id, page=1, post_id=None):
    """
    Download a thread.
    """
    # Download 1st page
    url = '{}/api/forum/threads/{}'.format(course.get_url(), thread_id)
    if post_id:
        url = '{}?post_id={}&position=after'.format(url, post_id)

    path = '{}/{}/{}.json'.format(threads_folder, thread_id, page)
    util.download(url, path, course.get_cookie_file())

    thread = util.read_json(path)
    download_images(course, threads_folder, thread)

    util.write_json(path, thread)

    # Download rest pages
    page = thread['start_page']
    num_page = thread['num_pages']

    if page < num_page:
        page += 1
        print 'thread page {}/{}'.format(page, num_page)

        post_id = get_next_post_id(thread['posts'])
        if post_id:
            download_thread(course, threads_folder, thread_id, page, post_id)

def _find_files(url, folder, cookie):
    """
    Recursively find all files in current page.
    :param url: A URL to given page.
    :param folder: A destination folder for this page.
    :param cookie: A cookie file used for downloading.
    :return: A list of files (URL, path) in current page.
    """
    files = []

    path = '{}/temp.html'.format(folder)
    util.download(url, path, cookie)

    page = util.read_file(path)
    util.remove(path)

    # recursively find all files in sub-folders
    pattern = r'<tr><td colspan="4"><a href="(.*?)">(.*?)</a>'
    for find in re.finditer(pattern, page, re.DOTALL):
        url = find.group(1)
        sub_folder = '{}/{}'.format(folder, find.group(2))
        files += _find_files(url, sub_folder, cookie)

    # find all files in this page
    pattern = r'<tr><td>(.*?)</td>.*?Embed.*?<a href="(.*?)\?.*?">Download</a>'
    for find in re.finditer(pattern, page, re.DOTALL):
        url = find.group(2)
        file_name = find.group(1)
        path = u'{}/{}'.format(folder, file_name)
        files.append((url, path))

    return files

def find_threads(course, forum_folder, forum_id):
    """
    Find all threads in current forum.
    Note: forum 0 has every thread!
    """
    # download the 1st page of given forum
    query = 'sort=firstposted&page=1'
    url = '{}/api/forum/forums/{}/threads?{}'
    url = url.format(course.get_url(), forum_id, query)
    path = forum_folder + '/temp.json'
    util.download(url, path, course.get_cookie_file())

    # download a huge page with all threads
    forum = util.read_json(path)
    num_threads = forum['total_threads']
    url += '&page_size={}'.format(num_threads)
    util.download(url, path, course.get_cookie_file())

    # add each thread's id to forum info
    threads = util.read_json(path)['threads']
    util.remove(path)

    path = forum_folder + '/info.json'
    forum = util.read_json(path)

    forum_threads = []
    for thread in reversed(threads):
        forum_threads.append({'id': thread['id']})

    forum['num_threads'] = num_threads
    forum['threads'] = forum_threads

    util.write_json(path, forum)

 def prepare(self):
     if "version" in self.settings:
         version = self.settings["version"]
         download(self.url % (version, version), self.zipfile)
         unzip(self.zipfile, 'temp')
     else:
         git_clone(self.repo, 'master', 'src')

def _download_old_quizzes(course, item, path):
    """
    Download old version in-video quizzes.
    """
    url = '{}/admin/quiz/quiz_load?quiz_id={}'
    url = url.format(course.get_url(), item['quiz']['parent_id'])
    util.download(url, path, course.get_cookie_file())
    util.write_json(path, util.read_json(path))

    def _check_setuptools(self):

        if self.check_module("setuptools"):
            return

        url, name = URLS['ez_setup']
        util.download(url)

        self.run_py(name)

    def download_stats(self):
        url = self.url + '/data/stats'
        path = self.info_folder + '/stats.html'
        util.download(url, path, self.cookie_file)

        content = util.read_file(path)
        pattern = r'<h1.*?</table>'
        content = re.search(pattern, content, re.DOTALL).group(0)
        util.write_file(path, content)

    def _check_pyparsing(self):

        if self.check_module("pyparsing"):
            return

        url, name = URLS['pyparsing']
        util.download(url, name)

        self.run_ez(name)

def prepare_negative_dataset(dataset_directory):
    negative_dataset_url = \
        'http://www.ics.uci.edu/~dramanan/papers/parse/people.zip'
    data_filepath = os.path.join(dataset_root,
                                 os.path.basename(negative_dataset_url))
    if not(os.path.exists(data_filepath)):
        download(negative_dataset_url, path=data_filepath)
    unzip(data_filepath, dataset_root)

    shutil.move(os.path.join(dataset_root, 'people_all'), dataset_directory)

    def _check_python(self):

        if os.path.exists(os.path.join(PYDIR, "python.exe")):
            return True

        url, name = URLS['python']
        util.download(url)

        print "Extracting the the python installer..."
        os.system('msiexec /a %s /qn TARGETDIR="%s"' % (name, PYDIR))

 def test_download(self):
   # Create a temporary file name, then delete it...
   # Of course, never do this in non-testing code!!
   f = tempfile.NamedTemporaryFile()
   f.close()
   # Make sure it doesn't exist.
   assert not os.path.exists(f.name)
   # Now download, using the deleted temporary file name.
   util.download(self.url, f.name)
   assert util.sha1file(f.name) == self.sha1_gtfs

def _load_from_url(url, to_filename):
    """ First downloads the connectome file to a file to_filename
    load it and return the reference to the connectome object
    
    Not tested.
    """
    
    from util import download    
    download(url, to_filename)
    return _load_from_cff(to_filename)

def download(url,name):
	downloads = __addon__.getSetting('downloads')
	if '' == downloads:
		xbmcgui.Dialog().ok(__scriptname__,__language__(30031))
		return
	stream = resolve(url)
	if stream:
		util.reportUsage(__scriptid__,__scriptid__+'/download')
		name+='.flv'
		util.download(__addon__,name,stream['url'],os.path.join(downloads,name))

 def prepare(self):
     if "version" in self.settings:
         version = self.settings["version"]
         download(self.url % (version), self.zipfile)
         unzip(self.zipfile, 'temp')
         cp('temp/imgui-%s/' % (version), 'temp/') # TODO: mv would be cleaner
     else:
         git_clone(self.repo, 'master', 'temp')
     if "patch" in self.settings:
         with cd('temp/'):
             patch(self.settings["patch"])