Python botocore.UNSIGNED属性代码示例

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def fix_s3_host(request, signature_version, region_name,
                default_endpoint_url='s3.amazonaws.com', **kwargs):
    """
    This handler looks at S3 requests just before they are signed.
    If there is a bucket name on the path (true for everything except
    ListAllBuckets) it checks to see if that bucket name conforms to
    the DNS naming conventions.  If it does, it alters the request to
    use ``virtual hosting`` style addressing rather than ``path-style``
    addressing.  This allows us to avoid 301 redirects for all
    bucket names that can be CNAME'd.
    """
    # By default we do not use virtual hosted style addressing when
    # signed with signature version 4.
    if signature_version is not botocore.UNSIGNED and \
            's3v4' in signature_version:
        return
    elif not _allowed_region(region_name):
        return
    try:
        switch_to_virtual_host_style(
            request, signature_version, default_endpoint_url)
    except InvalidDNSNameError as e:
        bucket_name = e.kwargs['bucket_name']
        logger.debug('Not changing URI, bucket is not DNS compatible: %s',
                     bucket_name) 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def download_file_from_s3(bucket_name: str, key: str, local_path: str) -> None:
    """
    Downloads file from S3 anonymously
    :param bucket_name: S3 Bucket name
    :param key: S3 File key name
    :param local_path: Local file path to download as
    """
    verify_ssl = get_verify_ssl()
    if not os.path.isfile(local_path):
        client = boto3.client(
            "s3", config=Config(signature_version=UNSIGNED), verify=verify_ssl
        )

        try:
            logger.info("Downloading S3 data file...")
            total = client.head_object(Bucket=bucket_name, Key=key)["ContentLength"]
            with ProgressPercentage(client, bucket_name, key, total) as Callback:
                client.download_file(bucket_name, key, local_path, Callback=Callback)
        except ClientError:
            raise KeyError(f"File {key} not available in {bucket_name} bucket.")

    else:
        logger.info(f"Reusing cached file {local_path}...") 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def assume_role(cls, role_arn, principal_arn, saml_response, duration=3600):
        ''' Assumes the desired role using the saml_response given. The response should be b64 encoded.
            Duration is in seconds
            :param role_arn: role amazon resource name
            :param principal_arn: principal name
            :param saml_response: SAML object to assume role with
            :param duration: session duration (default: 3600)
            :return: AWS session token
        '''
        # Assume role with new SAML
        conn = boto3.client('sts', config=client.Config(signature_version=botocore.UNSIGNED, user_agent=cls.USER_AGENT, region_name=None))
        aws_session_token = conn.assume_role_with_saml(
            RoleArn=role_arn,
            PrincipalArn=principal_arn,
            SAMLAssertion=saml_response,
            DurationSeconds=duration,
            
        )
        return aws_session_token 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def upload_export_tarball(self, realm: Optional[Realm], tarball_path: str) -> str:
        def percent_callback(bytes_transferred: Any) -> None:
            sys.stdout.write('.')
            sys.stdout.flush()

        # We use the avatar bucket, because it's world-readable.
        key = self.avatar_bucket.Object(os.path.join("exports", generate_random_token(32),
                                                     os.path.basename(tarball_path)))

        key.upload_file(tarball_path, Callback=percent_callback)

        session = botocore.session.get_session()
        config = Config(signature_version=botocore.UNSIGNED)

        public_url = session.create_client('s3', config=config).generate_presigned_url(
            'get_object',
            Params={
                'Bucket': self.avatar_bucket.name,
                'Key': key.key,
            },
            ExpiresIn=0,
        )
        return public_url 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def get_s3_client(unsigned=True):
    """Return a boto3 S3 client with optional unsigned config.

    Parameters
    ----------
    unsigned : Optional[bool]
        If True, the client will be using unsigned mode in which public
        resources can be accessed without credentials. Default: True

    Returns
    -------
    botocore.client.S3
        A client object to AWS S3.
    """
    if unsigned:
        return boto3.client('s3', config=Config(signature_version=UNSIGNED))
    else:
        return boto3.client('s3') 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def test_only_dynamodb_calls_are_traced():
    """Test only a single subsegment is created for other AWS services.

    As the pynamodb patch applies the botocore patch as well, we need
    to ensure that only one subsegment is created for all calls not
    made by PynamoDB. As PynamoDB calls botocore differently than the
    botocore patch expects we also just get a single subsegment per
    PynamoDB call.
    """
    session = botocore.session.get_session()
    s3 = session.create_client('s3', region_name='us-west-2',
                               config=Config(signature_version=UNSIGNED))
    try:
        s3.get_bucket_location(Bucket='mybucket')
    except ClientError:
        pass

    subsegments = xray_recorder.current_segment().subsegments
    assert len(subsegments) == 1
    assert subsegments[0].name == 's3'
    assert len(subsegments[0].subsegments) == 0 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def iterate_datasets(bucket_name, config, prefix, suffix, start_date, end_date, func, unsafe, sources_policy):
    manager = Manager()
    queue = manager.Queue()

    s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED))
    bucket = s3.Bucket(bucket_name)
    logging.info("Bucket : %s prefix: %s ", bucket_name, str(prefix))
    # safety = 'safe' if not unsafe else 'unsafe'
    worker_count = cpu_count() * 2

    processess = []
    for i in range(worker_count):
        proc = Process(target=worker, args=(config, bucket_name, prefix, suffix, start_date, end_date, func, unsafe, sources_policy, queue,))
        processess.append(proc)
        proc.start()

    for obj in bucket.objects.filter(Prefix=str(prefix)):
        if (obj.key.endswith(suffix)):
            queue.put(obj.key)

    for i in range(worker_count):
        queue.put(GUARDIAN)

    for proc in processess:
        proc.join() 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def worker(parse_only, queue):
    s3 = boto3.resource("s3", config=Config(signature_version=UNSIGNED))
    dc = datacube.Datacube()
    idx = dc.index

    while True:
        try:
            url = queue.get(timeout=60)
            if url == STOP_SIGN:
                break
            logging.info("Processing {} {}".format(url, current_process()))
            index_dataset(idx, s3, url, parse_only)
            queue.task_done()
        except Empty:
            break
        except EOFError:
            break 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def fetch_autoclaved_bucket(dst_dir, bucket_date):
    print("Fetch bucket")
    dst_bucket_dir = os.path.join(dst_dir, bucket_date)
    if not os.path.exists(dst_bucket_dir):
        os.makedirs(dst_bucket_dir)
    client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
    resource = boto3.resource("s3", config=Config(signature_version=UNSIGNED))

    prefix = "autoclaved/jsonl.tar.lz4/{}/".format(bucket_date)
    paginator = client.get_paginator("list_objects")
    for result in paginator.paginate(Bucket="ooni-data", Delimiter="/", Prefix=prefix):
        for f in result.get("Contents", []):
            fkey = f.get("Key")
            dst_pathname = os.path.join(dst_bucket_dir, os.path.basename(fkey))
            try:
                s = os.stat(dst_pathname)
                if s.st_size == f.get("Size"):
                    continue
            except Exception:  # XXX maybe make this more strict. It's FileNotFoundError on py3 and OSError on py2
                pass
            print("[+] Downloading {}".format(dst_pathname))
            resource.meta.client.download_file("ooni-data", fkey, dst_pathname) 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def get_s3_resource():
    session = boto3.session.Session()
    if session.get_credentials() is None:
        # Use unsigned requests.
        s3_resource = session.resource("s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED))
    else:
        s3_resource = session.resource("s3")
    return s3_resource 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def _get_credentials(self):
        """Get credentials by calling assume role."""
        kwargs = self._assume_role_kwargs()
        # Assume role with web identity does not require credentials other than
        # the token, explicitly configure the client to not sign requests.
        config = AioConfig(signature_version=UNSIGNED)
        async with self._client_creator('sts', config=config) as client:
            return await client.assume_role_with_web_identity(**kwargs) 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def _choose_signer(self, operation_name, signing_type, context):
        signing_type_suffix_map = {
            'presign-post': '-presign-post',
            'presign-url': '-query'
        }
        suffix = signing_type_suffix_map.get(signing_type, '')

        signature_version = self._signature_version
        if signature_version is not botocore.UNSIGNED and not \
                signature_version.endswith(suffix):
            signature_version += suffix

        handler, response = await self._event_emitter.emit_until_response(
            'choose-signer.{0}.{1}'.format(
                self._service_id.hyphenize(), operation_name),
            signing_name=self._signing_name, region_name=self._region_name,
            signature_version=signature_version, context=context)

        if response is not None:
            signature_version = response
            # The suffix needs to be checked again in case we get an improper
            # signature version from choose-signer.
            if signature_version is not botocore.UNSIGNED and not \
                    signature_version.endswith(suffix):
                signature_version += suffix

        return signature_version 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def _choose_signer(self, operation_name, signing_type):
        """
        Allow setting the signature version via the choose-signer event.
        A value of `botocore.UNSIGNED` means no signing will be performed.

        :param operation_name: The operation to sign.
        :param signing_type: The type of signing that the signer is to be used
            for.
        :return: The signature version to sign with.
        """
        signing_type_suffix_map = {
            'presign-post': '-presign-post',
            'presign-url': '-query'
        }
        suffix = signing_type_suffix_map.get(signing_type, '')

        signature_version = self._signature_version
        if signature_version is not botocore.UNSIGNED and not \
                signature_version.endswith(suffix):
            signature_version += suffix

        handler, response = self._event_emitter.emit_until_response(
            'choose-signer.{0}.{1}'.format(self._service_name, operation_name),
            signing_name=self._signing_name, region_name=self._region_name,
            signature_version=signature_version)

        if response is not None:
            signature_version = response
            # The suffix needs to be checked again in case we get an improper
            # signature version from choose-signer.
            if signature_version is not botocore.UNSIGNED and not \
                    signature_version.endswith(suffix):
                signature_version += suffix

        return signature_version 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def getCrawlIndex(_param):

    if not _param: #get the most recent index from common crawl
        bucket  = 'commoncrawl'
        s3      = boto3.client('s3', config=Config(signature_version=UNSIGNED))

        #verify bucket
        contents    = []
        prefix      = 'cc-index/collections/CC-MAIN-'
        botoArgs    = {'Bucket': bucket, 'Prefix': prefix}

        while True:

            objects = s3.list_objects_v2(**botoArgs)

            for obj in objects['Contents']:
                key = obj['Key']

                if 'indexes' in key:
                    cIndex = key.split('/indexes/')[0].split('/')
                    cIndex = cIndex[len(cIndex)-1]

                    if str(cIndex) not in contents:
                        contents.append(str(cIndex))

            try:
                botoArgs['ContinuationToken'] = objects['NextContinuationToken']
            except KeyError:
                break

        if contents:
            _param = contents[-1]

    return _param 

# 需要导入模块: import botocore [as 别名]
# 或者: from botocore import UNSIGNED [as 别名]
def set_operation_specific_signer(context, signing_name, **kwargs):
    """ Choose the operation-specific signer.

    Individual operations may have a different auth type than the service as a
    whole. This will most often manifest as operations that should not be
    authenticated at all, but can include other auth modes such as sigv4
    without body signing.
    """
    auth_type = context.get('auth_type')

    # Auth type will be None if the operation doesn't have a configured auth
    # type.
    if not auth_type:
        return

    # Auth type will be the string value 'none' if the operation should not
    # be signed at all.
    if auth_type == 'none':
        return botocore.UNSIGNED

    if auth_type.startswith('v4'):
        signature_version = 'v4'
        if signing_name == 's3':
            signature_version = 's3v4'

        # If the operation needs an unsigned body, we set additional context
        # allowing the signer to be aware of this.
        if auth_type == 'v4-unsigned-body':
            context['payload_signing_enabled'] = False

        return signature_version