Python requests_aws4auth.AWS4Auth方法代码示例

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def connect_es(endpoint):
    # Handle aws auth for es
    session = boto3.Session()
    credentials = session.get_credentials()
    awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, session.region_name, 'es',
                       session_token=credentials.token)
    print('Connecting to the ES Endpoint: {endpoint}'.format(endpoint=endpoint))
    try:
        es_client = Elasticsearch(
            hosts=[{'host': endpoint, 'port': 443}],
            use_ssl=True,
            verify_certs=True,
            http_auth=awsauth,
            connection_class=RequestsHttpConnection)
    except Exception as e:
        print("Unable to connect to {endpoint}:".format(endpoint=endpoint), e)
    else:
        print('Connected to elasticsearch')
        return es_client 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def _make_extra_headers(self, url: str, headers: Mapping[str, Any]) -> Mapping[str, Any]:
        """
        Returns the headers we should pass for this request.  This includes both the AWS v4 signature
        authentication ones as well as the Sec-WebSocket-* ones (which might vary and mess up the signatures used in
        authentication)
        """
        raw_request: bytes = self._get_raw_request_for(url=url, header=headers, **self.extra_websocket_options)
        request: PreparedRequest = self._parse_raw_request(raw_request)
        before_auth = set([k.lower() for k in request.headers.keys()])
        # we're always supposed to exclude these but AWS4Auth will include them with include_hdrs='*', so just delete
        # from the fake PreparedRequest we pass to AWS4Auth
        for k in set(request.headers.keys()):
            if k.lower() in ('connection', 'x-amzn-trace-id'):
                del request.headers[k]
        # usually mutates request (contract is to return it though, so cover our bases)
        request = self.aws4auth(request)
        # keep header if added by websocket client or aws4auth
        extra_headers = dict()
        for k, v in request.headers.items():
            if k.lower() not in before_auth or k.lower().startswith('sec-websocket'):
                extra_headers[k] = v
        return extra_headers 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def __get_aws_es_connection(self):
        """
        Create a new Elasticsearch connection to an AWS endpoint and return it.
        :return: An Elasticsearch connection.
        """
        from requests_aws4auth import AWS4Auth
        awsauth = AWS4Auth(config.aws_key_id, config.aws_secret_key, config.aws_default_region, "es")
        if config.http_proxy_enabled:
            proxies = {
                "http": config.http_proxy,
                "https": config.http_proxy,
            }
        else:
            proxies = {}
        return Elasticsearch(
            hosts=[{"host": config.es_host, "port": config.es_port}],
            http_auth=awsauth,
            use_ssl=True,
            verify_certs=True,
            connection_class=WsElasticsearchConnection,
            proxies=proxies,
        ) 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def __init__(self, *, aws_access_key_id: str, aws_secret_access_key: str, service_region: str,
                 service_name: str = 'neptune-db', extra_aws4auth_options: Mapping[str, Any] = {},
                 extra_websocket_options: Mapping[str, Any] = {}) -> None:
        # override any of these extra options (because we rely on their behavior)
        extra_aws4auth_options = dict(extra_aws4auth_options)
        extra_aws4auth_options.update(include_hdrs='*', raise_invalid_date=True)
        self.aws4auth = AWS4Auth(
            aws_access_key_id, aws_secret_access_key, service_region, service_name, **extra_aws4auth_options)
        super().__init__(extra_websocket_options=extra_websocket_options) 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def _parse_raw_request(cls, raw_request: bytes) -> PreparedRequest:
        """
        ok, this is kind of janky, but AWS4Auth is meant to work with requests, so expects a PreparedRequest
        """
        body: Optional[str] = None
        headers, body = raw_request.decode('utf-8').split('\r\n\r\n', 1)
        # strip the trailing \r\n if present
        if len(body) == 0:
            body = None
        elif body.endswith('\r\n'):
            body = body[:-2]
        # hi!  if you get here looking for folded headers, that's obsolete and we ought not be generating them
        method_et_al, headers = headers.split('\r\n', 1)
        headers_as_dict: Mapping[str, str] = \
            dict([(k.strip(), v.strip()) for k, v in [h.split(':', 1) for h in headers.split('\r\n')]])
        # this is a little janky, really should be one or more spaces
        method, path_et_al, version = method_et_al.split(' ', 2)
        # this is very sketchy looking but I promise that we don't care about the host, port, or scheme here
        url = 'https://nope/' + path_et_al
        req = PreparedRequest()
        req.prepare_method(method)
        req.prepare_url(url, {})
        req.prepare_headers(headers_as_dict)
        req.prepare_body(data=body, files=None)
        # don't req.prepare_content_length, we already had that in headers surely
        return req 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def getAccountIds(access_key, secret_key, domain, cb_alias):

    boto3Session = boto3.Session()
    region = boto3Session.region_name

    tailorEndpoint = 'https://' + domain + '/accounts/ids'
    auth = AWS4Auth(access_key, secret_key, region, 'execute-api')
    headers = {
        'host': domain,
        'accountCbAlias': cb_alias
    }
    tailorResponse = requests.get(tailorEndpoint, auth=auth, headers=headers)

    return json.loads(tailorResponse.content)['accountIds'] 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def lambda_handler(event, context):
    '''
    lambda handler to create index templates 
    '''    
    status = True
    host = os.environ['ElasticsearchEndpoint']

    logger.info('REQUEST RECEIVED:\n {}'.format(event))
    logger.info('REQUEST RECEIVED:\n {}'.format(context))
    
    try:   
        if event['RequestType'] == 'Create':
            logger.info('CREATE!')

            session = boto3.Session()
            credentials = session.get_credentials()
            awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, session.region_name, 'es', session_token=credentials.token)

            es = Elasticsearch([{'host': host, 'port': 443}], http_auth = awsauth, use_ssl = True, verify_certs=True, connection_class = RequestsHttpConnection)
            result = es.indices.put_template(name='jobtemplate', body=job_index_template)
            status1 = result.get('acknowledged', False)
            result = es.indices.put_template(name='metrictemplate', body=metric_index_template)
            status2 = result.get('acknowledged', False)
            if (status1 == False or status2 == False):
                send(event, context, "FAILED", { "Message": "Resource creation failed!" }, None)
            else:
                send(event, context, "SUCCESS", { "Message": "Resource creation successful!" }, None)
        elif event['RequestType'] == 'Update':
            logger.info('UPDATE!')
            send(event, context, "SUCCESS", { "Message": "Resource update successful!" }, None)
        elif event['RequestType'] == 'Delete':
            logger.info('DELETE!')
            send(event, context, "SUCCESS", { "Message": "Resource deletion successful!" }, None)
        else:
            logger.info('FAILED!')
            send(event, context, "FAILED", { "Message": "Unexpected event received from CloudFormation" }, None)
    
    except Exception as e:
        message = "Unexected error creating mapping: {}".format(e)
        send(event, context, "FAILED", { "Message": message }, None)
            
    return status 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def __init__(self, config_d, test=False):
        super(EventStore, self).__init__(config_d)
        # es kwargs
        kwargs = {}

        # es kwargs > hosts
        hosts = []
        configured_hosts = config_d.get("hosts", config_d.get("servers"))
        for host in configured_hosts:
            if not isinstance(host, dict):
                o = urllib.parse.urlparse(host)
                host = {k: v for k, v in (('host', o.hostname),
                                          ('port', o.port),
                                          ('url_prefix', o.path)) if v}
                if o.scheme == "https" or o.port == 443:
                    host['use_ssl'] = True
            hosts.append(host)
        kwargs['hosts'] = hosts

        # es kwargs > verify_certs
        if any(host.get("use_ssl") for host in hosts):
            kwargs['verify_certs'] = True

        # es kwargs > http_auth (for AWS)
        aws_auth = config_d.get("aws_auth")
        if aws_auth:
            kwargs["connection_class"] = RequestsHttpConnection
            try:
                kwargs['http_auth'] = AWS4Auth(aws_auth['access_id'],
                                               aws_auth['secret_key'],
                                               aws_auth['region'],
                                               'es')
            except KeyError:
                raise ImproperlyConfigured("access_id, secret_key or region missing "
                                           "in aws_auth config")

        self._es = Elasticsearch(**kwargs)
        self.use_mapping_types = None

        self.index = config_d['index']
        self.read_index = config_d.get('read_index', self.index)
        self.kibana_base_url = config_d.get('kibana_base_url', None)
        self.kibana_index_pattern_uuid = config_d.get('kibana_index_pattern_uuid')
        self.index_settings = {
            "index.mapping.total_fields.limit": config_d.get("index.mapping.total_fields.limit", 2000),
            "number_of_shards": config_d.get("number_of_shards", 1),
            "number_of_replicas": config_d.get("number_of_replicas", 0)
        }
        self.test = test
        self.version = None 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def _initialize(self):
        """
        Initialize a connection to an ES cluster and creates an index template if it does not exist.
        """
        if not self._initialized:
            http_auth = None
            if self._access_key and self._secret_key and self._aws_region:
                http_auth = AWS4Auth(self._access_key, self._secret_key, self._aws_region, "es")
            elif self._access_key and self._secret_key:
                http_auth = (self._access_key, self._secret_key)
            else:
                logger.warn("Connecting to Elasticsearch without HTTP auth")

            self._client = connections.create_connection(
                hosts=[{"host": self._host, "port": self._port}],
                http_auth=http_auth,
                use_ssl=self._use_ssl,
                verify_certs=True,
                connection_class=RequestsHttpConnection,
                timeout=ELASTICSEARCH_DEFAULT_CONNECTION_TIMEOUT,
            )

            # Create a second connection with a timeout of 60s vs 10s.
            # For some reason the PUT template API can take anywhere between
            # 10s and 30s on the test cluster.
            # This only needs to be done once to initialize the index template
            connections.create_connection(
                alias=ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS,
                hosts=[{"host": self._host, "port": self._port}],
                http_auth=http_auth,
                use_ssl=self._use_ssl,
                verify_certs=True,
                connection_class=RequestsHttpConnection,
                timeout=ELASTICSEARCH_TEMPLATE_CONNECTION_TIMEOUT,
            )

            try:
                force_template_update = ELASTICSEARCH_FORCE_INDEX_TEMPLATE_UPDATE.lower() == "true"
                self._client.indices.get_template(self._index_prefix)
                LogEntry.init(
                    self._index_prefix,
                    self._index_settings,
                    skip_template_init=not force_template_update,
                )
            except NotFoundError:
                LogEntry.init(self._index_prefix, self._index_settings, skip_template_init=False)
            finally:
                try:
                    connections.remove_connection(ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS)
                except KeyError as ke:
                    logger.exception(
                        "Elasticsearch connection not found to remove %s: %s",
                        ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS,
                        ke,
                    )

            self._initialized = True 

# 需要导入模块: import requests_aws4auth [as 别名]
# 或者: from requests_aws4auth import AWS4Auth [as 别名]
def lambda_handler(event, context):
    s3 = boto3.client('s3')
    credentials = boto3.Session().get_credentials()
    awsauth = AWS4Auth( credentials.access_key, 
                        credentials.secret_key, 
                        globalVars['awsRegion'], 
                        globalVars['service'], 
                        session_token=credentials.token
                    )

    logger.info("Received event: " + json.dumps(event, indent=2))
 
    try:    
        bucket = event['Records'][0]['s3']['bucket']['name']
        key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'])
       
        # Get documet (obj) form S3
        obj = s3.get_object(Bucket=bucket, Key=key)

    except Exception as e:
        logger.error('ERROR: {0}'.format( str(e) ) )
        logger.error('ERROR: Unable able to GET object:{0} from S3 Bucket:{1}. Verify object exists.'.format(key, bucket) )

    if (key.endswith('.gz')) or (key.endswith('.tar.gz')):
        mycontentzip = gzip.GzipFile(fileobj=BytesIO(obj['Body'].read())).read()
        lines = mycontentzip.decode("utf-8").replace("'", '"')
        # print('unziped file')
    else:
        lines = obj['Body'].read().decode("utf-8").replace("'", '"')
        
    logger.info('SUCCESS: Retreived object from S3')

    # Split (S3 object/Log File) by lines
    lines = lines.splitlines()
    if (isinstance(lines, str)):
        lines = [lines]

    # Index each line to ES Domain
    indexName = globalVars['esIndexPrefix'] + str( datetime.date.today().year ) + '-' + str( datetime.date.today().month )
    es_Url = globalVars['esHosts'].get('prod') + '/' + indexName + '/' + globalVars['esIndexDocType']
    
    docData = {}
    docData['objectKey']        = str(key)
    docData['createdDate']      = str(obj['LastModified'])
    docData['content_type']     = str(obj['ContentType'])
    docData['content_length']   = str(obj['ContentLength'])

    for line in lines:
        docData['content'] = str(line)
        indexDocElement(es_Url, awsauth, docData )
    logger.info('SUCCESS: Successfully indexed the entire doc into ES')