Python http.MediaIoBaseDownload类代码示例

def get(service):
    # User can be prompted to input file name(using raw_input) that needs to be be downloaded.
    file_name = raw_input("Enter file name to be downloaded:")
    try:
        # Get Metadata
        req = service.objects().get(
            bucket=_BUCKET_NAME,
            object=file_name,
            fields='bucket,name,metadata(my-key)',

        )
        resp = req.execute()
        print json.dumps(resp, indent=2)
        # Get Payload Data
        req = service.objects().get_media(
            bucket=_BUCKET_NAME,
            object=file_name,
        )
        # The Bytes I/O object may be replaced with any io.Base instance.
        fh = io.BytesIO()
        downloader = MediaIoBaseDownload(fh, req, chunksize=1024 * 1024)  # show progress at download
        done = False
        while not done:
            status, done = downloader.next_chunk()
            if status:
                print 'Download %d%%.' % int(status.progress() * 100)
            print 'Download Complete'

        fo = decrypt(fh.getvalue(),key)
        fi = open(file_name,'wb')
        fi.write(fo)
        # fh.getvalue() contains downloaded content.Decrypt the file and save it to onto your local machine
        print json.dumps(resp, indent=2)
    except client.AccessTokenRefreshError:
        print ("Error in the credentials")

    def pull(bucket, local_file, metadata_only=False):
        objname = os.path.basename(local_file)
        k = None
        try:
            k = JBoxGS.connect().objects().get(bucket=bucket,
                                               object=objname).execute()
        except HttpError as err:
            if err._get_reason() != 'Not Found':
                raise(err)
            else:
                return None

        if not metadata_only:
            req = JBoxGS.connect().objects().get_media(bucket=bucket,
                                                       object=objname)
            fh = open(local_file, "wb")
            downloader = MediaIoBaseDownload(fh, req, chunksize=JBoxGS.CHUNK_SIZE*1024*1024)
            done = False
            num_retries = 0
            while not done:
                try:
                    _, done = downloader.next_chunk()
                except HttpError, err:
                    num_retries += 1
                    if num_retries > JBoxGS.MAX_RETRIES:
                        fh.close()
                        os.remove(local_file)
                        raise
                    if err.resp.status in JBoxGS.RETRYABLE_ERRORS:
                        backoff = min(JBoxGS.BACKOFF_FACTOR ** (num_retries - 1),
                                      JBoxGS.MAX_BACKOFF)
                        sleep(backoff + random())
                    else:
                        sleep(JBoxGS.SLEEP_TIME)
                except:

    def __init__(self, row: List[str], drive_service):
        self.number = int(row[0])
        content_url = urlparse(row[1])
        summary_url = urlparse(row[4])
        repetition_material = urlparse(row[7])

        if len(summary_url.scheme) > 0:
            self.summary_url = summary_url.geturl()
        else:
            self.summary_url = None

        if len(repetition_material.scheme) > 0:
            self.repetition_material = repetition_material.geturl()
        else:
            self.repetition_material = None

        if len(content_url.scheme) > 0:
            file_id = content_url.path.split('/')[-1]
            if file_id == 'open':
                file_id = parse_qs(content_url.query)['id']

            request = drive_service.files().export_media(fileId=file_id,
                                                         mimeType='text/plain')
            fh = io.BytesIO()
            downloader = MediaIoBaseDownload(fh, request)
            done = False
            while done is False:
                status, done = downloader.next_chunk()
            self.content = fh.getvalue().decode('utf-8')
            self.content_type = None
        else:
            self.content = None
            self.content_type = row[1]

    def pull(bucket, local_file, metadata_only=False):
        objname = os.path.basename(local_file)
        k = None
        try:
            k = JBoxGS.connect().objects().get(bucket=bucket,
                                               object=objname).execute()
        except HttpError as err:
            if err._get_reason() != 'Not Found':
                raise(err)
            else:
                return None

        if metadata_only:
            return k
        else:
            req = JBoxGS.connect().objects().get_media(bucket=bucket,
                                                       object=objname)

            fh = open(local_file, "wb")
            downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
            done = False
            try:
                while not done:
                    _, done = downloader.next_chunk()
            finally:
                fh.close()
                if not done:
                    os.remove(local_file)
            return k

 def load_config(self):
     try:
         results = self._gdrive.files().list(
             corpora="user",
             q="parents='%s' and name='%s'" % (self._album_id, GdriveAlbum.CONFIG_FILE),
             pageSize=1,
             spaces='drive').execute()
     except Error:
         results = {}
     items = results.get('files', [])
     if len(items) == 0:
         self._config_file_id = ''
         self._config = {} # TODO new config with default values
         return
     config_item = items[0]
     self._config_file_id=config_item["id"]
     request = self._gdrive.files().get_media(fileId=self._config_file_id)
     fh = io.BytesIO()
     downloader = MediaIoBaseDownload(fh, request)
     while True:
         _, done = downloader.next_chunk()
         if done: break
     config = json.loads(fh.getvalue())
     if "albums" not in config:
         config["albums"] = []
     if "pictures" not in config:
         config["pictures"] = []
     self._config = config

    def download(self, mime_type=None):
        """Download the content of the file from Google Drive

        Args:
            mime_type: the mime type of the file to download.
                      see here:
                      https://developers.google.com/drive/v3/web/manage-downloads#downloading_google_documents

        Returns:
            The content of the file
        """

        if mime_type is None:
            download_type = MimeTypes.get_download_type(self.meta_data['mimeType'])
        else:
            download_type = mime_type

        req = self.service.files().export_media(fileId=self.meta_data['id'],
                                                mimeType=download_type)
        data = io.BytesIO()
        downloader = MediaIoBaseDownload(data, req)
        done = False
        while not done:
            _, done = downloader.next_chunk()

        data.seek(0)
        self.content = data.read()

        return self.content

def download(service, bucketName, objectName, filename):
    """
    :type service: Resource
    :type bucketName: basestring
    :type objectName: basestring
    :type filename: basestring
    """

    print 'Building download request...'
    f = file(filename, 'w')
    request = service.objects().get_media(bucket=bucketName,
                                          object=objectName)

    media = MediaIoBaseDownload(f, request, chunksize=CHUNKSIZE)

    print 'Downloading bucket: %s object: %s to file: %s' % (bucketName,
                                                             objectName,
                                                             filename)

    progressless_iters = 0
    done = False

    while not done:
        error = None
        try:
            progress, done = media.next_chunk()
            if progress:
                print_with_carriage_return(
                    'Download %d%%.' % int(progress.progress() * 100))
        except HttpError, err:
            error = err
            if err.resp.status < 500:
                raise
        except RETRYABLE_ERRORS, err:
            error = err

  def test_media_io_base_download(self):
    self.request.http = HttpMockSequence([
      ({'status': '200',
        'content-range': '0-2/5'}, '123'),
      ({'status': '200',
        'content-range': '3-4/5'}, '45'),
    ])
    self.assertEqual(True, self.request.http.follow_redirects)

    download = MediaIoBaseDownload(
        fd=self.fd, request=self.request, chunksize=3)

    self.assertEqual(self.fd, download._fd)
    self.assertEqual(3, download._chunksize)
    self.assertEqual(0, download._progress)
    self.assertEqual(None, download._total_size)
    self.assertEqual(False, download._done)
    self.assertEqual(self.request.uri, download._uri)

    status, done = download.next_chunk()

    self.assertEqual(self.fd.getvalue(), '123')
    self.assertEqual(False, done)
    self.assertEqual(3, download._progress)
    self.assertEqual(5, download._total_size)
    self.assertEqual(3, status.resumable_progress)

    status, done = download.next_chunk()

    self.assertEqual(self.fd.getvalue(), '12345')
    self.assertEqual(True, done)
    self.assertEqual(5, download._progress)
    self.assertEqual(5, download._total_size)

def download_file(service, item, download_folder='./data/', overwrite=False):
    if not is_downloadable(item):
        return False

    local_path = download_folder + item['path']

    if os.path.isfile(local_path) and not overwrite:
        if file_md5(local_path) == item['md5Checksum']:
            return False
        else:
            print("Corrupt file '%s'" % local_path)

    mkdir_p(os.path.dirname(local_path))

    with open(local_path, "wb") as destination:
        request = service.files().get_media(fileId=item['id'])
        downloader = MediaIoBaseDownload(destination, request)
        done = False

        while done is False:
            _, done = downloader.next_chunk()

    if file_md5(local_path) != item['md5Checksum']:
        raise Exception("Download for '%s' failed, wrong checksum" % local_path)

    return True

    def download_file(self, file_id, write_path, page_num=None, print_details=True, output_type=None):
        file_metadata = self._files.get(fileId=file_id, fields='name, id, mimeType, modifiedTime, size').execute(num_retries=self._max_retries)

        file_title = file_metadata['name']
        modified_date = datetime.strptime(str(file_metadata['modifiedTime']), '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=utc).astimezone(timezone('Asia/Singapore')).replace(tzinfo=None)

        return_data = None

        if file_metadata['mimeType'] == 'application/vnd.google-apps.spreadsheet':
            assert page_num is not None

            download_url = 'https://docs.google.com/spreadsheets/d/%s/export?format=csv&gid=%i' % (file_id, page_num)
            resp, content = self._service._http.request(download_url)

            if resp.status == 200:

                if output_type is not None:
                    assert output_type in ('dataframe', 'list')
                    from io import BytesIO

                    with BytesIO(content) as file_buffer:
                        if output_type == 'list':
                            import unicodecsv as csv
                            return_data = list(csv.reader(file_buffer))
                        elif output_type == 'dataframe':
                            import pandas as pd
                            return_data = pd.read_csv(file_buffer)

                else:
                    with open(write_path, 'wb') as write_file:
                        write_file.write(content)

                logging_string = '[Drive] Downloaded %s [%s]. Last Modified: %s' % (file_title, file_id, modified_date)
            else:
                raise HttpError(resp, content)

        else:
            request = self._files.get_media(fileId=file_id)

            with open(write_path, 'wb') as write_file:
                downloader = MediaIoBaseDownload(write_file, request)

                done = False
                while done is False:
                    status, done = downloader.next_chunk()

            file_size = humanize.naturalsize(int(file_metadata['size']))
            logging_string = '[Drive] Downloaded %s [%s] (%s). Last Modified: %s' % (file_title, file_id, file_size, modified_date)

        if print_details:
            print '\t' + logging_string

        if self._logger is not None:
            self._logger.info(logging_string)

        return return_data

 def download_file(self, local_file_path, bucket_name, storage_file_path):
     f = file(local_file_path, 'wb')
     request = self.service.objects().get_media(bucket=bucket_name, object=storage_file_path)
     media = MediaIoBaseDownload(f, request)
     base_name = os.path.basename(local_file_path)
     done = False
     while not done:
         progress, done = media.next_chunk()
         if progress:
             print('{0} is download {1}/100'.format(base_name, int(100 * progress.progress())))

 def stream(self, uri, file=None):
     file = file or self._get(uri, fields='files(id)')
     if file:
         output = tempfile.TemporaryFile()
         request = self.service.files().get_media(fileId=file.id)
         downloader = MediaIoBaseDownload(output, request)
         done = False
         while not done:
             status, done = downloader.next_chunk()
         output.seek(0)
         return output

def main():
    drive_service = driveClient()

    file_id = '1pE6ZovOBy4koVwWvKwE_Pt-7whVsYRKj'
    request = drive_service.files().get_media(fileId=file_id)
    fh = io.FileIO('faw.csv', 'wb')
    downloader = MediaIoBaseDownload(fh, request)
    done = False
    while done is False:
        status, done = downloader.next_chunk()
        print(f'Download {int(status.progress() * 100)}')

 def get_contents_to_fileobj(self, key, fileobj_to_store_to):
     key = self.format_key_for_backend(key)
     self.log.debug("Starting to fetch the contents of: %r to %r", key, fileobj_to_store_to)
     with self._object_client(not_found=key) as clob:
         req = clob.get_media(bucket=self.bucket_name, object=key)
         download = MediaIoBaseDownload(fileobj_to_store_to, req, chunksize=CHUNK_SIZE)
         done = False
         while not done:
             status, done = download.next_chunk()
             if status:
                 self.log.debug("Download of %r: %d%%", key, status.progress() * 100)
         return self._metadata_for_key(clob, key)

def get_file(name, id):
    import io
    download_service = discovery.build('drive', 'v3', developerKey=_dev_key, http=decorator.http())
    request = download_service.files().get_media(fileId=id)
    fh = io.BytesIO()
    downloader = MediaIoBaseDownload(fh, request)
    done = False
    while not done:
        status, done = downloader.next_chunk()

    fh.seek(0)
    return fh.read()