Python BucketManager.list方法代码示例

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def bucket_list():
    q = Auth(access_key, secret_key)
    bucket = BucketManager(q)
    ret, eof, info = bucket.list(bucket_name, limit=4)
    for item in ret.get('items', []):
        print item
    ret, eof, info = bucket.list(bucket_name, limit=100)
    print(info)

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
    def _list_remote_bucket(self):
        """
        traverse the buckt through the Qiniu API and return a list of keys
        and timestamp
        :except no connection is established
        :return: a dict mapping key to upload time
        """
        key_list = {}
        big_file = {}
        done = False
        marker = None

        bucket = BucketManager(self.auth)

        while not done:
            res, done, _ = bucket.list(self.bucketname, marker=marker,
                                       limit=self.BATCH_LIMIT)
            if not res:
                self.logger('ERROR',
                            'could not establish connection with cloud. Exit.')
                sys.exit(1)
            marker = res.get('marker')
            for resource in res['items']:
                key_list[resource['key']] = resource['putTime']
                big_file[resource['key']] = resource['fsize'] \
                    if resource['fsize'] > self.download_size_threshold * 2 \
                    else 0
        return key_list, big_file

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
    def teardown_class(cls):
        """Delete all files in the test bucket.
        """
        storage = QiniuPrivateStorage(
            bucket_name=get_qiniu_config('QINIU_PRIVATE_BUCKET_NAME'),
            bucket_domain=get_qiniu_config('QINIU_PRIVATE_BUCKET_DOMAIN'),
        )
        auth = storage.auth
        bucket = BucketManager(auth)

        while True:
            ret, eof, info = bucket.list(storage.bucket_name, limit=100)

            if ret is None:
                print(info)
                break

            for item in ret['items']:
                name = item['key']
                if six.PY2:
                    name = name.encode('utf-8')
                ret, info = bucket.delete(storage.bucket_name, name)
                if ret is None:
                    print(info)
            if eof:
                break

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
 def list(self, bucket_name, prefix, limit=5):
     bucket = BucketManager(self.mauth)
     delimiter = None
     marker = None
     ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter)
     for item in ret.get('items'):
         print "{Name: %s Size:%s Hash:%s}" % (item['key'], item['fsize'], item['hash'])

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
class QiniuWrapper():

    policy = {'returnBody': '{"key": $(key), "type": $(mimeType), "name": $(fname), "size": $(fsize), "hash": $(etag)}'}
    bucket_name = QINIU_BUCKET_NAME
    domain = 'http://%s.qiniudn.com/' % bucket_name

    def __init__(self):
        self.q = Auth(QINIU_ACCESS_KEY, QINIU_SECRET_KEY)
        self.bucket_manager = BucketManager(self.q)

    def get_upload_token(self, key, expires=3600):
        return self.q.upload_token(self.bucket_name, key, expires, self.policy)

    def upload_file(self, key, filename, mime_type="application/octet-stream"):
        '''
        上传文件到七牛，如果指定的key对应的文件在七牛上已经存在, 会覆盖原来七牛上的文件
        '''
        ret, info = put_file(self.get_upload_token(key), key, filename, mime_type=mime_type, check_crc=True)
        if info.status_code != 200:
            return (False, info)
        return (True, info)

    def upload_stream(self, key, input_stream, data_size, mime_type="application/octet-stream"):
        '''
        上传文件到七牛，如果指定的key对应的文件在七牛上已经存在, 会覆盖原来七牛上的文件
        '''
        ret, info = put_stream(self.get_upload_token(key), key, input_stream, data_size, mime_type=mime_type, check_crc=True)
        if info.status_code != 200:
            return (False, info)
        return (True, info)

    def move(self, old_key, new_key):
        ret, info = self.bucket_manager.move(self.bucket_name, old_key, self.bucket_name, new_key)
        if info.status_code != 200:
            return (False, info)
        return (True, info)

    def delete(self, key):
        ret, info = self.bucket_manager.delete(self.bucket_name, key)
        if info.status_code != 200:
            return (False, info)
        return (True, info)

    def batch_delete(self, keys):
        '''
        keys = ['key1', 'key2', 'key3']
        '''
        ops = build_batch_delete(self.bucket_name, keys)
        ret, info = self.bucket_manager.batch(ops)
        if info.status_code != 200:
            return (False, info)
        return (True, info)

    def list(self, prefix=None, limit=1000, marker=None):
        return self.bucket_manager.list(self.bucket_name, prefix=prefix, marker=marker, limit=limit)

    @classmethod
    def get_url(cls, key):
        return cls.domain + key

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def qiniu_file(pre_text, limit):
    q = qiniu_q()
    bucket = BucketManager(q)
    bucket_name = 'love-travel'
    ret, eof, info = bucket.list(bucket_name, prefix=pre_text, delimiter="/", limit=limit)
    if len(ret["items"]) > 0 :
        return {'result' : True, 'files' : ret["items"]}
    else:
        return {'result' : False}

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def bucket_list():
    bm = BucketManager(qa())
    ret = bm.list('image', None, None, 10, None)
    if ret and ret[0] and ret[0].get('items') and len(ret[0].get('items')) > 0:
        imgs = ret[0].get('items')
        marker = ret[0].get('marker')
        print(marker)
        if imgs:
            for x in imgs:
                print('ll.append("%s")' % x['key'])
    print(ret)

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def get_all_uploaded_files():
  sys.stdout.write('Retrieve uploaded file in qiniu...\n')
  sys.stdout.flush()
  bucket = BucketManager(auth)
  marker = None
  eof = False
  uploaded_files = []
  while not eof:
    ret, eof, info = bucket.list(bucket_name, marker=marker)
    marker = ret.get('marker', None)
    for item in ret['items']:
      uploaded_files.append(item['key'])
  return uploaded_files

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def list_wrong_files(bucket_name, bucket=None, prefix=None, limit=None):
    if bucket is None:
        bucket = BucketManager(q)
    marker = None
    eof = False
    files_key = []
    while eof is False:
        ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit)
        marker = ret.get('marker', None)
        for item in ret['items']:
            if item['fsize'] == 85:
                files_key.append(item['key'])
    return files_key

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
class EntryMonitor:
    def __init__(self):
        self._lock = threading.RLock()
        self._stop_cond = threading.Condition(self._lock)
        self._stop = False

        self.qiniu = Auth(config.qiniu_ak, config.qiniu_sk)
        self.blog_bucket = BucketManager(self.qiniu)
        self.blog_prefix = "raw/"

        self._thread = threading.Thread(target = self.run, name="blog_entry_monitor")
        self._thread.daemon = True
        self._thread.start()

    def __del__(self):
        with self._lock:
            self._stop = True
            self._stop_cond.notify()
        self._thread.join()

    def run(self):
        with self._lock:
            while not self._stop:
                try:
                    local_list = {}
                    for root, _, files in os.walk(config.entry_dir):
                        for f in files:
                            local_list[f] = False

                    ret, _, info = self.blog_bucket.list(config.qiniu_bucket, prefix = "raw/")
                    file_list = eval(info.text_body)["items"]
                    print "find file list %s" % file_list
                    for f in file_list:
                        if f["key"].startswith(self.blog_prefix) and f["mimeType"] == "text/markdown":
                            file_name = f["key"][len(self.blog_prefix):]
                            if file_name not in local_list:
                                entryService.download_file(file_name, f["key"])
                                entryService.add_entry(True, 'raw/entry/' + file_name)
                            else:
                                local_list[file_name] = True

                    for f in local_list.keys():
                        if not local_list[f]:
                            print "remove entry %s" % f
                            entryService.delete_entry(os.path.abspath('raw/entry/' + f))
                            os.remove('raw/entry/' + f)
                except Exception, e:
                    print "***************** %s *****************" % e
                self._stop_cond.wait(10)

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def list_all(bucket_name, bucket=None, prefix="", limit=100):
    rlist=[]
    if bucket is None:
        bucket = BucketManager(q)
    marker = None
    eof = False
    while eof is False:
        ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit)
        marker = ret.get('marker', None)
        for item in ret['items']:
            rlist.append(item["key"])
    if eof is not True:
        # 错误处理
        #print "error"
        pass
    return rlist

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def list_all(bucket_name, bucket=None, prefix=None, limit=None):
    photos = []
    if bucket is None:
        bucket = BucketManager(q)
    marker = None
    eof = False
    while eof is False:
        ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit)
        marker = ret.get('marker', None)
        for item in ret['items']:
            print(item['key'])
            photos.append(item['key'])
    if eof is not True:
        # 错误处理
        pass
    return photos

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
def insert_all(bucket_name, bucket=None, prefix=None, limit=None):
    if bucket is None:
        bucket = BucketManager(q)
    marker = None
    eof = False
    while eof is False:
        buck = bucket.list(
            bucket_name, prefix=prefix, marker=marker, limit=limit)
        ret = buck[0]
        eof = buck[1]
        marker = ret.get('marker', None)
        for item in ret['items']:
            print(item['key'])
            store(item['key'])
    if eof is not True:
        pass

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
class QiniuUploader:
    __is_initialized = False
    __access_key = ""
    __secret_key = ""
    __bucket_name = ""

    def __init__(self):
        pass

    def set_auth(self, access_key: str, secret_key: str, bucket_name: str):
        self.__access_key = access_key
        self.__secret_key = secret_key
        self.__bucket_name = bucket_name
        self.__auth = Auth(self.__access_key, self.__secret_key)
        self.__bucket = BucketManager(self.__auth)
        self.__is_initialized = True
        return True

    def upload(self, file_from: str, file_to: str, process_callback):
        if not self.__is_initialized:
            print("Have not been initialized!")
            return None
        print("Qiniu Ready Upload [%s]" % file_from)
        names = file_to.split("/")
        file_to = names[len(names) - 1]
        token = self.__auth.upload_token(self.__bucket_name, file_to)
        ret, info = put_file(token, file_to, file_from, progress_handler=process_callback)
        assert ret['key'] == file_to
        assert ret['hash'] == etag(file_from)
        return info

    def remove(self, file_name):
        if not self.__is_initialized:
            print("Have not been initialized!")
            return None
        ret, info = self.__bucket.delete(self.__bucket_name, file_name)
        return info

    def list_files(self) -> list:
        ret = self.__bucket.list(self.__bucket_name)
        if ret[2].status_code != 200:
            return ret
        return_var = []
        for item in ret[0]['items']:
            assert isinstance(item, dict)
            return_var.append(item)
        return return_var

# 需要导入模块: from qiniu import BucketManager [as 别名]
# 或者: from qiniu.BucketManager import list [as 别名]
class QiniuOperation(OperationPackage):

    def init(self):
        self.q = Auth(
            self.get_config('qiniu_access_key'),
            self.get_config('qiniu_secret_key'),
        )
        self.bucket = BucketManager(self.q)

    def token(self, filename):
        return self.q.upload_token(
            self.get_config('qiniu_bucket'),
            filename,
        )

    def generate_file_hash(self, data):
        fhash = etag_stream(StringIO(data))
        if isinstance(fhash, bytes):
            fhash = fhash.decode('ascii')
        return fhash

    def list_remote(self):
        rep = self.bucket.list(self.get_config('qiniu_bucket'))
        assert rep[1]
        ret = []
        for element in rep[0]['items']:
            ret.append(
                (element['key'], element['hash']),
            )
        return ret

    def create_file(self):
        token = self.token(self.fname)
        put_file(token, self.fname, self.fpath)
        return self.fname

    def update_file(self, old_fhash):
        token = self.token(self.fname_with_hash)
        put_file(token, self.fname_with_hash, self.fpath)
        return self.fname_with_hash

    def resource_url(self, fname, hash_tag):
        return self.get_config('qiniu_base_url') + fname