Python s3boto3.S3Boto3Storage方法代码示例

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def run(self, id_filename):
        s3 = S3Boto3Storage().connection
        obj = s3.Object(self.output_bucket, id_filename)
        arxiv_id_str = obj.get()["Body"].read().decode("utf-8")
        arxiv_ids = [s.strip() for s in arxiv_id_str.split() if s.strip()]

        # We can't access database inside our gevent pool because of max
        # connections, so first figure out which IDs we actually want to
        # render.
        arxiv_ids, source_paths = self.filter_unrenderable_ids(arxiv_ids)

        # Stagger the starting of jobs a bit so we don't break Hyper.sh
        def slow_arxiv_ids():
            for arxiv_id in arxiv_ids:
                yield arxiv_id
                time.sleep(0.1)

        pool = Pool(self.concurrency)
        manifest = pool.imap_unordered(self.render, slow_arxiv_ids(), source_paths)
        # Failed renders are None
        manifest = (obj for obj in manifest if obj)
        # Read the iterator, starting the actual processing
        manifest = list(manifest)
        self.write_manifest(manifest) 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def ready(self):
        from django.core.files.storage import default_storage
        from storages.backends.s3boto3 import S3Boto3Storage
        from django import forms

        from .forms import S3FileInputMixin

        if isinstance(default_storage, S3Boto3Storage) and \
                S3FileInputMixin not in forms.ClearableFileInput.__bases__:
            forms.ClearableFileInput.__bases__ = \
                (S3FileInputMixin,) + forms.ClearableFileInput.__bases__

        elif S3FileInputMixin in forms.ClearableFileInput.__bases__:
            forms.ClearableFileInput.__bases__ = tuple(
                cls for cls in forms.ClearableFileInput.__bases__
                if cls is not S3FileInputMixin
            )

        checks.register(storage_check, checks.Tags.security, deploy=True) 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def serve_document_from_s3(document, request):
    # Skip this hook if not using django-storages boto3 backend.
    if not issubclass(get_storage_class(), S3Boto3Storage):
        return

    # Send document_served signal.
    document_served.send(sender=get_document_model(), instance=document,
                         request=request)

    # Get direct S3 link.
    file_url = document.file.url

    # Generate redirect response and add never_cache headers.
    response = redirect(file_url)
    del response['Cache-control']
    add_never_cache_headers(response)
    return response 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def write_manifest(self, manifest):
        s3 = S3Boto3Storage().connection
        manifest_json = json.dumps(manifest, indent=2)
        s3.Object(self.output_bucket, "manifest.json").put(Body=manifest_json) 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def get_manifest():
    """
    Download and parse manifest from arxiv S3 bucket.
    """
    connection = S3Boto3Storage().connection
    obj = connection.Object("arxiv", "src/arXiv_src_manifest.xml")
    s = obj.get(RequestPayer="requester")["Body"].read()
    return parse_manifest(s) 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def download_tarball(key, local_filename):
    connection = S3Boto3Storage().connection
    bucket = connection.Bucket("arxiv")
    bucket.download_file(key, local_filename, {"RequestPayer": "requester"}) 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def setUp(self):
        self.storage = s3boto3.S3Boto3Storage()
        self.storage._connections.connection = mock.MagicMock() 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def test_location_leading_slash(self):
        msg = (
            "S3Boto3Storage.location cannot begin with a leading slash. "
            "Found '/'. Use '' instead."
        )
        with self.assertRaises(ImproperlyConfigured, msg=msg):
            s3boto3.S3Boto3Storage(location='/') 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def test_override_settings(self):
        with override_settings(AWS_LOCATION='foo1'):
            storage = s3boto3.S3Boto3Storage()
            self.assertEqual(storage.location, 'foo1')
        with override_settings(AWS_LOCATION='foo2'):
            storage = s3boto3.S3Boto3Storage()
            self.assertEqual(storage.location, 'foo2') 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def test_override_class_variable(self):
        class MyStorage1(s3boto3.S3Boto3Storage):
            location = 'foo1'

        storage = MyStorage1()
        self.assertEqual(storage.location, 'foo1')

        class MyStorage2(s3boto3.S3Boto3Storage):
            location = 'foo2'

        storage = MyStorage2()
        self.assertEqual(storage.location, 'foo2') 

# 需要导入模块: from storages.backends import s3boto3 [as 别名]
# 或者: from storages.backends.s3boto3 import S3Boto3Storage [as 别名]
def test_override_init_argument(self):
        storage = s3boto3.S3Boto3Storage(location='foo1')
        self.assertEqual(storage.location, 'foo1')
        storage = s3boto3.S3Boto3Storage(location='foo2')
        self.assertEqual(storage.location, 'foo2')