Python error.ResponseError方法代码示例

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def __initialize_object_store(self):

        # Initialize minioClient with an endpoint and access/secret keys.
        self.client = Minio(endpoint=self.endpoint.netloc,
                            access_key=self.access_key,
                            secret_key=self.secret_key,
                            secure=self.secure)

        # Make a bucket with the make_bucket API call.
        try:
            if not self.client.bucket_exists(self.bucket):
                self.client.make_bucket(self.bucket)
        except BucketAlreadyOwnedByYou as ex:
            self.log.warning("Object Storage bucket already owned by you", exc_info=True)
            raise ex from ex
        except BucketAlreadyExists as ex:
            self.log.warning("Object Storage bucket already exists", exc_info=True)
            raise ex from ex
        except ResponseError as ex:
            self.log.error("Object Storage error", exc_info=True)
            raise ex from ex

        return self.client 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def upload_minio(artifacts):
    minio_host = os.environ['MINIO_HOST']
    minioClient = Minio(minio_host+':9000', access_key='minio', secret_key='minio123', secure=False)

    for a in artifacts:
        try:
           file_stat = os.stat(a)
           file_data = open(a, 'rb')
           minioClient.put_object('artifacts', package_name()+'/'+os.path.basename(a), file_data, file_stat.st_size, content_type='application/vnd.dcos.universe.repo+json;charset=utf-8;version=v4')
        except ResponseError as err:
           print(err) 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def handler(event, context):
  if event['data']['EventType'] == "s3:ObjectCreated:Put" : 
        tf = tempfile.NamedTemporaryFile(delete=False)
        bucket = event['data']['Key'].split('/')[0]
        filename = event['data']['Key'].split('/')[1]
      
        # Fetching source file from Minio
        try:
            print('Fetching file')
            client.fget_object(bucket, filename, tf.name)
        except ResponseError as err:
            print('Error fetching file')
            print err

        # OCR text extraction performed by Tika
        print 'Sending file to Tika'
        parsed = parser.from_file(tf.name, 'http://tika-tika-server:80/tika')
        ocrdata = json.dumps(parsed, ensure_ascii=True)

        # MongoDB document insertion 
        db = mongo['ocr']
        result = db.processed.insert_one(parsed)
        print 'Document Saved!'
        print('Document proccessed: {0}'.format(result.inserted_id))

        # move OCRd file to done bucket 
        try:
            # Copy from input bucket to done bucket
            fullpath = 'input/' + filename 
            client.copy_object('done', filename, fullpath)
            # Remove from input bucket
            client.remove_object('input', filename)
        except ResponseError as err:
            print err
  else:
       print "Minio file deletion event"
      
  return "OCR Finished" 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def thumbnail(event, context):
    bucket = os.path.dirname(event['data']['Key'])
    _, file_extension = os.path.splitext(event['data']['Key'])
    filename = os.path.basename(event['data']['Key'])

    print file_extension.upper()

    if file_extension.upper() != ".JPEG":
        return "Not a picture"

    if event['data']['EventType'] == "s3:ObjectCreated:Put" and bucket == 'foobar':

        tf = tempfile.NamedTemporaryFile(delete=False)
        tf_thumb = tempfile.NamedTemporaryFile(delete=False)

        try:
            client.fget_object(bucket, filename, tf.name)
        except ResponseError as err:
            print err

        size=(120,120)
        img = Image.open(tf.name)
        img.thumbnail(size)
        img.save(tf_thumb.name, "JPEG")

        # puts the thumbnail in a thumbnail bucket
        thumb_name = filename + '.thumb'
        try:
            client.fput_object('thumb',thumb_name,tf_thumb.name)
        except ResponseError as err:
            print err

    else:
        print "Minio file deletion event"

    return "Thumbnail creation triggered" 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def handle(self, *args, **options):
        bucket_name = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME

        try:
            client.client.make_bucket(bucket_name)
        except BucketAlreadyOwnedByYou:  # pragma: no cover
            self.stderr.write(f"Bucket: {bucket_name} is already owned by you!")
        except BucketAlreadyExists:  # pragma: no cover
            self.stderr.write(f"Bucket: {bucket_name} already exists!")
        except ResponseError:  # pragma: no cover
            raise
        else:
            self.stdout.write(f"Created bucket: {bucket_name}") 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def check_external_services():
    """Called after initialization. Use it to validate settings"""

    print('Checking AMQP connection')

    try:
        parameters = AMQP_CONNECTION_PARAMETERS
        connection = pika.BlockingConnection(parameters)
        connection.channel()
        connection.close()
    except AMQPError as _error:
        raise RuntimeError('AMQP connection failed')

    if DOCUMENTS_API['ENABLED']:
        print('Documents API enabled. Checking Minio connection')

        try:
            minio = Minio(MINIO['ENDPOINT'],
                          access_key=MINIO['ACCESS_KEY'],
                          secret_key=MINIO['SECRET_KEY'],
                          secure=MINIO['USE_SSL'])

            _objects = minio.list_buckets()

            if not minio.bucket_exists(MINIO['BUCKET_NAME']):
                print('Minio bucket doesn\'t exist. Creating it')
                try:
                    minio.make_bucket(MINIO['BUCKET_NAME'])
                except ResponseError as _error:
                    print(_error)
                    raise RuntimeError('Minio bucket creation failed')

        except MinioError as _error:
            raise RuntimeError('Minio connection failed')

    if KEYCLOAK['ENABLED']:
        print('Keycloak enabled. Checking connection')

        try:
            list_users(get_token())
        except Exception as _error:
            raise RuntimeError('Keycloak connection failed')

    if EMAIL['ENABLED']:
        print('Email sending enabled. Checking connection')

        try:
            with smtplib.SMTP(host=EMAIL['SMTP_SERVER_HOST'],
                              port=EMAIL['SMTP_SERVER_PORT']) as server:
                server.noop()
        except Exception as error:
            print(error)
            raise RuntimeError('SMTP Connection failed') 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def _save_to_store(self, checkpoint_dir):
        """
        save_to_store() uploads the policy checkpoint, gifs and videos to the S3 data store. It reads the checkpoint state files and
        uploads only the latest checkpoint files to S3. It is used by the trainer in Coach when used in the distributed mode.
        """
        try:
            # remove lock file if it exists
            self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value)

            # Acquire lock
            self.mc.put_object(self.params.bucket_name, SyncFiles.LOCKFILE.value, io.BytesIO(b''), 0)

            state_file = CheckpointStateFile(os.path.abspath(checkpoint_dir))
            if state_file.exists():
                ckpt_state = state_file.read()
                checkpoint_file = None
                for root, dirs, files in os.walk(checkpoint_dir):
                    for filename in files:
                        if filename == CheckpointStateFile.checkpoint_state_filename:
                            checkpoint_file = (root, filename)
                            continue
                        if filename.startswith(ckpt_state.name):
                            abs_name = os.path.abspath(os.path.join(root, filename))
                            rel_name = os.path.relpath(abs_name, checkpoint_dir)
                            self.mc.fput_object(self.params.bucket_name, rel_name, abs_name)

                abs_name = os.path.abspath(os.path.join(checkpoint_file[0], checkpoint_file[1]))
                rel_name = os.path.relpath(abs_name, checkpoint_dir)
                self.mc.fput_object(self.params.bucket_name, rel_name, abs_name)

            # upload Finished if present
            if os.path.exists(os.path.join(checkpoint_dir, SyncFiles.FINISHED.value)):
                self.mc.put_object(self.params.bucket_name, SyncFiles.FINISHED.value, io.BytesIO(b''), 0)

            # upload Ready if present
            if os.path.exists(os.path.join(checkpoint_dir, SyncFiles.TRAINER_READY.value)):
                self.mc.put_object(self.params.bucket_name, SyncFiles.TRAINER_READY.value, io.BytesIO(b''), 0)

            # release lock
            self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value)

            if self.params.expt_dir and os.path.exists(self.params.expt_dir):
                for filename in os.listdir(self.params.expt_dir):
                    if filename.endswith((".csv", ".json")):
                        self.mc.fput_object(self.params.bucket_name, filename, os.path.join(self.params.expt_dir, filename))

            if self.params.expt_dir and os.path.exists(os.path.join(self.params.expt_dir, 'videos')):
                for filename in os.listdir(os.path.join(self.params.expt_dir, 'videos')):
                        self.mc.fput_object(self.params.bucket_name, filename, os.path.join(self.params.expt_dir, 'videos', filename))

            if self.params.expt_dir and os.path.exists(os.path.join(self.params.expt_dir, 'gifs')):
                for filename in os.listdir(os.path.join(self.params.expt_dir, 'gifs')):
                        self.mc.fput_object(self.params.bucket_name, filename, os.path.join(self.params.expt_dir, 'gifs', filename))

        except ResponseError as e:
            print("Got exception: %s\n while saving to S3", e) 

# 需要导入模块: from minio import error [as 别名]
# 或者: from minio.error import ResponseError [as 别名]
def load_from_store(self):
        """
        load_from_store() downloads a new checkpoint from the S3 data store when it is not available locally. It is used
        by the rollout workers when using Coach in distributed mode.
        """
        try:
            state_file = CheckpointStateFile(os.path.abspath(self.params.checkpoint_dir))

            # wait until lock is removed
            while True:
                objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.LOCKFILE.value)

                if next(objects, None) is None:
                    try:
                        # fetch checkpoint state file from S3
                        self.mc.fget_object(self.params.bucket_name, state_file.filename, state_file.path)
                    except Exception as e:
                        continue
                    break
                time.sleep(10)

            # Check if there's a finished file
            objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.FINISHED.value)

            if next(objects, None) is not None:
                try:
                    self.mc.fget_object(
                        self.params.bucket_name, SyncFiles.FINISHED.value,
                        os.path.abspath(os.path.join(self.params.checkpoint_dir, SyncFiles.FINISHED.value))
                    )
                except Exception as e:
                    pass

            # Check if there's a ready file
            objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.TRAINER_READY.value)

            if next(objects, None) is not None:
                try:
                    self.mc.fget_object(
                        self.params.bucket_name, SyncFiles.TRAINER_READY.value,
                        os.path.abspath(os.path.join(self.params.checkpoint_dir, SyncFiles.TRAINER_READY.value))
                    )
                except Exception as e:
                    pass

            checkpoint_state = state_file.read()
            if checkpoint_state is not None:
                objects = self.mc.list_objects_v2(self.params.bucket_name, prefix=checkpoint_state.name, recursive=True)
                for obj in objects:
                    filename = os.path.abspath(os.path.join(self.params.checkpoint_dir, obj.object_name))
                    if not os.path.exists(filename):
                        self.mc.fget_object(obj.bucket_name, obj.object_name, filename)

        except ResponseError as e:
            print("Got exception: %s\n while loading from S3", e)