Python MediaIoBaseDownload.next_chunk方法代码示例

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
    def test_media_io_base_download(self):
        self.request.http = HttpMockSequence(
            [({"status": "200", "content-range": "0-2/5"}, "123"), ({"status": "200", "content-range": "3-4/5"}, "45")]
        )

        download = MediaIoBaseDownload(fh=self.fh, request=self.request, chunksize=3)

        self.assertEqual(self.fh, download.fh_)
        self.assertEqual(3, download.chunksize_)
        self.assertEqual(0, download.progress_)
        self.assertEqual(None, download.total_size_)
        self.assertEqual(False, download.done_)
        self.assertEqual(self.request.uri, download.uri_)

        status, done = download.next_chunk()

        self.assertEqual(self.fh.getvalue(), "123")
        self.assertEqual(False, done)
        self.assertEqual(3, download.progress_)
        self.assertEqual(5, download.total_size_)
        self.assertEqual(3, status.resumable_progress)

        status, done = download.next_chunk()

        self.assertEqual(self.fh.getvalue(), "12345")
        self.assertEqual(True, done)
        self.assertEqual(5, download.progress_)
        self.assertEqual(5, download.total_size_)

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
  def test_media_io_base_download(self):
    self.request.http = HttpMockSequence([
      ({'status': '200',
        'content-range': '0-2/5'}, '123'),
      ({'status': '200',
        'content-range': '3-4/5'}, '45'),
    ])

    download = MediaIoBaseDownload(
        fd=self.fd, request=self.request, chunksize=3)

    self.assertEqual(self.fd, download._fd)
    self.assertEqual(3, download._chunksize)
    self.assertEqual(0, download._progress)
    self.assertEqual(None, download._total_size)
    self.assertEqual(False, download._done)
    self.assertEqual(self.request.uri, download._uri)

    status, done = download.next_chunk()

    self.assertEqual(self.fd.getvalue(), '123')
    self.assertEqual(False, done)
    self.assertEqual(3, download._progress)
    self.assertEqual(5, download._total_size)
    self.assertEqual(3, status.resumable_progress)

    status, done = download.next_chunk()

    self.assertEqual(self.fd.getvalue(), '12345')
    self.assertEqual(True, done)
    self.assertEqual(5, download._progress)
    self.assertEqual(5, download._total_size)

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def pull_file(service, file_id, src_path, dst_path):
    request = service.files().get_media(fileId=file_id)
    with open(dst_path, 'w') as f:
        downloader = MediaIoBaseDownload(f, request)
        (status, done) = downloader.next_chunk()
        while done is False:
            logging.info('downloading... {:.2f}%'.format(
                status.progress() * 100))
            (status, done) = downloader.next_chunk()

    logging.info('{} <- {}'.format(dst_path, src_path))

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
  def test_media_io_base_download_retries_5xx(self):
    self.request.http = HttpMockSequence([
      ({'status': '500'}, ''),
      ({'status': '500'}, ''),
      ({'status': '500'}, ''),
      ({'status': '200',
        'content-range': '0-2/5'}, '123'),
      ({'status': '503'}, ''),
      ({'status': '503'}, ''),
      ({'status': '503'}, ''),
      ({'status': '200',
        'content-range': '3-4/5'}, '45'),
    ])

    download = MediaIoBaseDownload(
        fd=self.fd, request=self.request, chunksize=3)

    self.assertEqual(self.fd, download._fd)
    self.assertEqual(3, download._chunksize)
    self.assertEqual(0, download._progress)
    self.assertEqual(None, download._total_size)
    self.assertEqual(False, download._done)
    self.assertEqual(self.request.uri, download._uri)

    # Set time.sleep and random.random stubs.
    sleeptimes = []
    download._sleep = lambda x: sleeptimes.append(x)
    download._rand = lambda: 10

    status, done = download.next_chunk(num_retries=3)

    # Check for exponential backoff using the rand function above.
    self.assertEqual([20, 40, 80], sleeptimes)

    self.assertEqual(self.fd.getvalue(), '123')
    self.assertEqual(False, done)
    self.assertEqual(3, download._progress)
    self.assertEqual(5, download._total_size)
    self.assertEqual(3, status.resumable_progress)

    # Reset time.sleep stub.
    del sleeptimes[0:len(sleeptimes)]

    status, done = download.next_chunk(num_retries=3)

    # Check for exponential backoff using the rand function above.
    self.assertEqual([20, 40, 80], sleeptimes)

    self.assertEqual(self.fd.getvalue(), '12345')
    self.assertEqual(True, done)
    self.assertEqual(5, download._progress)
    self.assertEqual(5, download._total_size)

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def processObject(client, itemname):
    try:
        #try to access json object in Google Compute Storage
        # Get Payload Data
        req = client.objects().get_media(
                    bucket = errorsBucket,
                    object=itemname)
        #store info whether a json-object exists in the bucket or not
        fileExists = True
        try:
            resp = req.execute()
        except:
            fileExists = False
            pass
            
        #continue only when the object exists
        if (fileExists):
            # The BytesIO object may be replaced with any io.Base instance.
            fh = io.BytesIO()
            #prepare for reading a json-object
            downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
            done = False
            while not done:
                status, done = downloader.next_chunk()
            #print (fh.getvalue())
            #load accessed json-object into dictionary
            jsonFile = fh.getvalue()
            #print(jsonFile)
            jf = open("../" + errorsBucket + "/" + itemname.encode(), 'w')
            jf.write(jsonFile)
            jf.close()
            #store error message into respective errors bucket
    except oauth2_client.AccessTokenRefreshError:
        pass

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
  def retrieve_content(
      self, bucket, path, transform=None, generation=None, **kwargs):
    """Retrieves the content at the specified path.

    Args:
      bucket: [string] The bucket to retrieve front.
      path: [string] The path to the content to retrieve from the bucket.
      generation: [long] Specifies version of object (or None for current).
      transform: [callable(string)] transform the downloaded bytes into
         something else (e.g. a JSON object). If None then the identity.

    Returns:
      transformed object.
    """
    self.logger.info('Retrieving path=%s from bucket=%s [generation=%s]',
                     path, bucket, generation)

    # Get Payload Data
    request = self.service.objects().get_media(
        bucket=bucket,
        object=path,
        generation=generation,
        **kwargs)

    data = io.BytesIO()
    downloader = MediaIoBaseDownload(data, request, chunksize=1024*1024)
    done = False
    while not done:
      status, done = downloader.next_chunk()
      if status:
        self.logger.debug('Download %d%%', int(status.progress() * 100))
    result = data.getvalue()
    return result if transform is None else transform(result)

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def processObject(client, itemname):
    try:
        #try to access json object in Google Compute Storage
        # Get Payload Data
        req = client.objects().get_media(
                    bucket = comm.jsonsDir,
                    object=itemname)
        #store info whether a json-object exists in the bucket or not
        fileExists = True
        try:
            resp = req.execute()
        except:
            fileExists = False
            pass
            
        #continue only when the object exists
        if (fileExists):
            # The BytesIO object may be replaced with any io.Base instance.
            fh = io.BytesIO()
            #prepare for reading a json-object
            downloader = MediaIoBaseDownload(fh, req, chunksize=1024*1024)
            done = False
            while not done:
                status, done = downloader.next_chunk()
            #load accessed json-object into dictionary
            jsonToDict = json.loads(fh.getvalue())#json.loads(fh.getvalue())#return value
            #print ("RETURNED VALUE: " + jsonToDict)
            doDownloadJob(jsonToDict, itemname)
    #store error message into respective errors bucket
    except oauth2_client.AccessTokenRefreshError:
        comm.printException(comm.pathToSaveDownloadErrors, errString="False credentials")
        pass

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def download_file(service, filename):
	result = []
	result = retrieve_all_files(service)
	for i in result :

		if i['title'] == filename:

			tmp = i['id']
			break
	f = open (filename, 'wb')

	request = service.files().get_media(fileId=tmp)
	media_request = MediaIoBaseDownload(f, request)



	while True:
		try:
			download_progress, done = media_request.next_chunk()
		except errors.HttpError, error:
			print 'An error occurred: %s' % error
			return
		#if download_progress:
		#	print 'Download Progress: %d%%' % int(download_progress.progress() * 100)
		if done:
			print 'Google download complete'
			return

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def File(gs_path, chunk_size=CHUNK_SIZE):
  """Download a file from the cloud, and return a file that's readable.

  Args:
    gs_path: Fully qualified gfs path, eg, 'gfs://bucket/path/to/FILE'.
    chunk_size: The chunk_size to download, defaults to CHUNK_SIZE.

  Returns:
    An IO stream to be read.
  """
  bucket_name, object_name = gs_path[5:].split('/', 1)
  logging.info('Downloading file: %s/%s', bucket_name, object_name)

  credentials = GoogleCredentials.get_application_default()
  service = discovery.build('storage', 'v1', credentials=credentials)
  request = service.objects().get_media(bucket=bucket_name, object=object_name)
  output = StringIO.StringIO()
  media = MediaIoBaseDownload(output, request, chunksize=chunk_size)
  done = False
  while not done:
    try:
      _, done = media.next_chunk()
    except HttpError, err:
      if err.resp.status < 500:
        raise
    except RETRYABLE_ERRORS, err:
      pass

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
    def get(self, remote_path, local_path, report_to=None):
        LOG.debug("Downloading %s from cloud storage (local path: %s)", remote_path, local_path)
        bucket, name = self._parse_url(remote_path)
        local_path = os.path.join(local_path, os.path.basename(remote_path))

        request = self.cloudstorage.objects().get_media(bucket=bucket, object=name)

        f = open(local_path, "w")
        try:
            media = MediaIoBaseDownload(f, request, chunksize=self.chunk_size)

            last_progress = 0
            done = False
            while not done:
                status, done = media.next_chunk()
                if status:
                    percentage = int(status.progress() * 100)
                    if percentage - last_progress >= self.report_interval:
                        if report_to:
                            report_to(status.resumable_progress, status.total_size)
                        last_progress = percentage
        finally:
            f.close()

        LOG.debug("Finished downloading %s", os.path.basename(local_path))
        return local_path

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def main():
    """Shows basic usage of the Google Drive API.

    Creates a Google Drive API service object and outputs the names and IDs
    for up to 10 files.
    """
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
    service = discovery.build('drive', 'v3', http=http)

    #file_id = '1cxgbJZKnysKOKBDg-ZbV1E3S4B-iAG7XY-1x7U8Yfsg' # For the grocery doc
    file_id = '18r3cUWKbMaWVYtNKLJjxZFHB2m7y1QJdkSPlrU197PA' # For the test doc

    request = service.files().export_media(fileId=file_id, mimeType='text/plain')

    response = request.execute()

    fh = open(FILE_LOC, 'wb')
    downloader = MediaIoBaseDownload(fh, request)
    done = False
    while done is False:
        status, done = downloader.next_chunk()
        print("Download %.2f%%." % (status.progress() * 100.0))


    fh.close()
    # Now read doc.txt to get information
    find_date()

    # Upload the file back
    update_request = service.files().update(fileId=file_id, media_body=FILE_LOC)

    update_response = update_request.execute()

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
def download(argv):
  bucket_name, object_name = argv[1][5:].split('/', 1)
  filename = argv[2]
  assert bucket_name and object_name

  service = get_authenticated_service(RO_SCOPE)

  print 'Building download request...'
  f = file(filename, 'w')
  request = service.objects().get_media(bucket=bucket_name,
                                        object=object_name)
  media = MediaIoBaseDownload(f, request, chunksize=CHUNKSIZE)

  print 'Downloading bucket: %s object: %s to file: %s' % (bucket_name,
                                                           object_name,
                                                           filename)

  progressless_iters = 0
  done = False
  while not done:
    error = None
    try:
      progress, done = media.next_chunk()
      if progress:
        print_with_carriage_return(
            'Download %d%%.' % int(progress.progress() * 100))
    except HttpError, err:
      error = err
      if err.resp.status < 500:
        raise
    except RETRYABLE_ERRORS, err:
      error = err

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
        def scan_item_contents(item):
            if item.get('mimType') == self.FOLDER_MIME:
                return

            shared = item.get('shared', None)
            if shared is None or shared:
                return
            # only process files
            file_id = item.get('id', None)
            name = item.get('name', None)
            available = True
            if file_id and name:
                f_path = create_temp_name(temp_dir, name)
                logger.warning("Processing file {}...{}".format(name, f_path))
                f = open(f_path, 'wb')
                try:
                    request = self.client.files().get_media(fileId=file_id)
                    downloader = MediaIoBaseDownload(f, request)
                    done = False
                    while done is False:
                        status, done = downloader.next_chunk()
                except Exception as ex:
                    logger.error("Unable to download file {}. {}".format(name, ex))
                    available = False
                f.close()
                if available:
                    try:
                        matches.extend(search_content(f_path, expressions))
                    except Exception as ex:
                        logger.error("Unable to parse content in file {}. {}".format(name, ex))

                try:
                    os.remove(f_path)
                except Exception as ex:
                    logger.error("Unable to clean up temprary file {}. {}".format(f_path, ex))

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
    def download(self, folder_service_callback):
        uploadFolder = self.helper.get_fileid_by_name(helper.UPLOADFOLDER)
        # get newest file download url
        downloadInfo = self.helper.get_newest_file_down_info(uploadFolder)
        progressless_iters = 0

        # download file
        print('Downloading latest backup ...')
        filename = folder_service_callback.getTempFolder() + downloadInfo['title']
        fh = io.FileIO(filename, 'wb')
        downloader = MediaIoBaseDownload(fh, downloadInfo['request'])
        done = False
        while not done:
            error = None
            try:
                status, done = downloader.next_chunk()
                print("Download %d%%." % int(status.progress() * 100), end="\r")

            except HttpError as err:
                error = err
                if err.resp.status < 500:
                    raise

            except self.RETRYABLE_ERRORS as err:
                error = err


            if error:
                progressless_iters += 1
                self.handle_progressless_iter(error, progressless_iters)
            else:
                progressless_iters = 0

        return filename

# 需要导入模块: from apiclient.http import MediaIoBaseDownload [as 别名]
# 或者: from apiclient.http.MediaIoBaseDownload import next_chunk [as 别名]
    def download_file(self,file_id, file_name):
        """Download a Drive file's content to the local filesystem.

            Args:
            service: Drive API Service instance.
            file_id: ID of the Drive file that will downloaded.
            file_name: used as name for to write content in.
        """
        fd = open(file_name,'w+')
        request = self._service.files().get_media(fileId=file_id)
        media_request = MediaIoBaseDownload(fd, request)

        while True:
            try:
                download_progress, done = media_request.next_chunk()
            except:
                print('An error occurred')
                fd.close()
                return
            if download_progress:
                print('Download Progress: %d%%' % int(download_progress.progress() * 100))
            if done:
                print('Download Complete')
                fd.close()
                return