Python S3_hook.S3Hook方法代碼示例

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def render_template(**context):
    """ Render HTML template using questions metadata from S3 bucket """

    hook = S3Hook(aws_conn_id="s3_connection")
    file_content = hook.read_key(
        key=S3_FILE_NAME, bucket_name=Variable.get("S3_BUCKET")
    )
    questions = json.loads(file_content)

    root = os.path.dirname(os.path.abspath(__file__))
    env = Environment(loader=FileSystemLoader(root))
    template = env.get_template("email_template.html")
    html_content = template.render(questions=questions)

    # Push rendered HTML as a string to the Airflow metadata database
    # to make it available for the next task

    task_instance = context["task_instance"]
    task_instance.xcom_push(key="html_content", value=html_content) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def copy_file_to_s3_staging(
        identifier,
        tsv_file_path,
        s3_bucket,
        aws_conn_id,
        media_prefix=DEFAULT_MEDIA_PREFIX,
        staging_prefix=STAGING_PREFIX,
):
    logger.info(f'Creating staging object in s3_bucket:  {s3_bucket}')
    s3 = S3Hook(aws_conn_id=aws_conn_id)
    file_name = os.path.split(tsv_file_path)[1]
    staging_object_prefix = _get_staging_object_prefix(
        identifier,
        media_prefix,
        staging_prefix
    )
    staging_key = _s3_join_path(staging_object_prefix, file_name)
    s3.load_file(tsv_file_path, staging_key, bucket_name=s3_bucket) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def get_staged_s3_object(
        identifier,
        s3_bucket,
        aws_conn_id,
        media_prefix=DEFAULT_MEDIA_PREFIX,
        staging_prefix=STAGING_PREFIX,
):
    s3 = S3Hook(aws_conn_id=aws_conn_id)
    staging_object_prefix = _get_staging_object_prefix(
        identifier,
        media_prefix,
        staging_prefix
    )
    key_list = s3.list_keys(s3_bucket, prefix=staging_object_prefix)
    assert len(key_list) == 1
    return key_list[0] 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def getSalesforceRecords(name, **kwargs):
    sf = get_salesforce_conn()
    formatted_name = "{}.json".format(name.lower())
    templates_dict = kwargs.get('templates_dict', {})
    fields = json.loads(templates_dict.get('fields', '[]'))
    query_string = "SELECT {0} FROM {1}".format(','.join(fields), name)
    print(query_string)
    response = sf.query_all(query_string)
    output = response['records']
    output = '\n'.join([json.dumps(flatten({boa.constrict(k): v\
                        for k, v in i.items()})) for i in output])

    with  NamedTemporaryFile("w") as f:
        f.write(output)
        s3_key = 'salesforce/{}'.format(formatted_name)
        s3 = S3Hook(s3_conn_id='INSERT_S3_CONN_ID_HERE')
        s3.load_file(
            filename=f.name,
            key=s3_key,
            bucket_name='INSERT_S3_BUCKET_NAME_HERE',
            replace=True
        )
        s3.connection.close()
        return s3_key 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def read_and_format(self):
        if self.schema_location.lower() == 's3':
                hook = S3Hook(self.s3_conn_id)
                # NOTE: In retrieving the schema, it is assumed
                # that boto3 is being used. If using boto, 
                # `.get()['Body'].read().decode('utf-8'))`
                # should be changed to
                # `.get_contents_as_string(encoding='utf-8'))`
                schema = (hook.get_key(self.origin_schema,
                                       bucket_name=
                                       '{0}'.format(self.s3_bucket))
                          .get()['Body'].read().decode('utf-8'))
                schema = json.loads(schema.replace("'", '"'))
        else:
            schema = self.origin_schema

        return schema 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def execute(self, context):
        """
        Executed by task_instance at runtime
        """
        mongo_conn = MongoHook(self.mongo_conn_id).get_conn()
        s3_conn = S3Hook(self.s3_conn_id)

        # Grab collection and execute query according to whether or not it is a pipeline
        collection = mongo_conn.get_database(self.mongo_db).get_collection(self.mongo_collection)
        results = collection.aggregate(self.mongo_query) if self.is_pipeline else collection.find(self.mongo_query)

        # Performs transform then stringifies the docs results into json format
        docs_str = self._stringify(self.transform(results))

        s3_conn.load_string(docs_str, self.s3_key, bucket_name=self.s3_bucket, replace=self.replace) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def execute(self, context):
        s3 = S3Hook(self.s3_conn_id)
        mongo = MongoHook(conn_id=self.mongo_conn_id)

        data = (s3
                .get_key(self.s3_key,
                         bucket_name=self.s3_bucket)
                .get_contents_as_string(encoding='utf-8'))

        docs = [json.loads(doc) for doc in data.split('\n')]

        self.method_mapper(mongo, docs) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def write_questions_to_s3():
    hook = S3Hook(aws_conn_id="s3_connection")
    hook.load_string(
        string_data=filter_questions(),
        key=S3_FILE_NAME,
        bucket_name=Variable.get("S3_BUCKET"),
        replace=True,
    ) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def execute(self, context):
        ga_conn = GoogleAnalyticsHook(self.google_analytics_conn_id)
        s3_conn = S3Hook(self.s3_conn_id)

        account_summaries = ga_conn.get_account_summaries()

        file_name = '/tmp/{key}.jsonl'.format(key=self.s3_key)
        with open(file_name, 'w') as ga_file:
            data = []
            for item in account_summaries.get('items', []):
                root_data_obj = {
                    'account_id': item['id'],
                    'brand': self.brand,
                    'space': self.space
                }

                for web_property in item.get('webProperties', []):
                    data_obj = {}
                    data_obj.update(root_data_obj)

                    data_obj['property_id'] = web_property['id']

                    for profile in web_property.get('profiles', []):
                        data_obj['profile_id'] = profile['id']
                        data_obj['profile_name'] = profile['name']
                        data.append(data_obj)

            json_data = '\n'.join([json.dumps(d) for d in data])
            ga_file.write(json_data)

        s3_conn.load_file(file_name, self.s3_key, self.s3_bucket, True)
        os.remove(file_name) 

# 需要導入模塊: from airflow.hooks import S3_hook [as 別名]
# 或者: from airflow.hooks.S3_hook import S3Hook [as 別名]
def execute(self, context):
        sf_conn = SalesforceHook(self.sf_conn_id).get_conn()

        logging.info(self.soql)
        query_results = sf_conn.bulk.__getattr__(self.object).query(self.soql)

        s3 = S3Hook(self.s3_conn_id)
        # One JSON Object Per Line
        query_results = [json.dumps(result, ensure_ascii=False) for result in query_results]
        query_results = '\n'.join(query_results)

        s3.load_string(query_results, self.s3_key, bucket_name=self.s3_bucket, replace=True)