本文整理汇总了Python中airflow.contrib.hooks.emr_hook.EmrHook类的典型用法代码示例。如果您正苦于以下问题：Python EmrHook类的具体用法？Python EmrHook怎么用？Python EmrHook使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了EmrHook类的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_create_job_flow_uses_the_emr_config_to_create_a_cluster
def test_create_job_flow_uses_the_emr_config_to_create_a_cluster(self):
client = boto3.client('emr', region_name='us-east-1')
hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
cluster = hook.create_job_flow({'Name': 'test_cluster'})
self.assertEqual(client.list_clusters()['Clusters'][0]['Id'],
cluster['JobFlowId'])示例2: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info('Terminating JobFlow %s', self.job_flow_id)
response = emr.terminate_job_flows(JobFlowIds=[self.job_flow_id])
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('JobFlow termination failed: %s' % response)
else:
logging.info('JobFlow with id %s terminated', self.job_flow_id)示例3: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info("Adding steps to %s", self.job_flow_id)
response = emr.add_job_flow_steps(JobFlowId=self.job_flow_id, Steps=self.steps)
if not response["ResponseMetadata"]["HTTPStatusCode"] == 200:
raise AirflowException("Adding steps failed: %s" % response)
else:
logging.info("Steps %s added to JobFlow", response["StepIds"])
return response["StepIds"]示例4: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id, emr_conn_id=self.emr_conn_id)
logging.info('Creating JobFlow')
response = emr.create_job_flow(self.job_flow_overrides)
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('JobFlow creation failed: %s' % response)
else:
logging.info('JobFlow with id %s created', response['JobFlowId'])
return response['JobFlowId']示例5: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
self.log.info('Adding steps to %s', self.job_flow_id)
response = emr.add_job_flow_steps(JobFlowId=self.job_flow_id, Steps=self.steps)
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('Adding steps failed: %s' % response)
else:
self.log.info('Steps %s added to JobFlow', response['StepIds'])
return response['StepIds']示例6: test_create_job_flow_extra_args
def test_create_job_flow_extra_args(self):
"""
Test that we can add extra arguments to the launch call.
This is useful for when AWS add new options, such as
"SecurityConfiguration" so that we don't have to change our code
"""
client = boto3.client('emr', region_name='us-east-1')
hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
# AmiVersion is really old and almost no one will use it anymore, but
# it's one of the "optional" request params that moto supports - it's
# coverage of EMR isn't 100% it turns out.
cluster = hook.create_job_flow({'Name': 'test_cluster',
'ReleaseLabel': '',
'AmiVersion': '3.2'})
cluster = client.describe_cluster(ClusterId=cluster['JobFlowId'])['Cluster']
# The AmiVersion comes back as {Requested,Running}AmiVersion fields.
self.assertEqual(cluster['RequestedAmiVersion'], '3.2')示例7: test_get_conn_returns_a_boto3_connection
def test_get_conn_returns_a_boto3_connection(self):
hook = EmrHook(aws_conn_id='aws_default')
self.assertIsNotNone(hook.get_conn().list_clusters())示例8: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info('Poking cluster %s' % self.job_flow_id)
return emr.describe_cluster(ClusterId=self.job_flow_id)示例9: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
self.log.info('Poking step %s on cluster %s', self.step_id, self.job_flow_id)
return emr.describe_step(ClusterId=self.job_flow_id, StepId=self.step_id)示例10: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
_log.info('Poking step {0} on cluster {1}'.format(self.step_id, self.job_flow_id))
return emr.describe_step(ClusterId=self.job_flow_id, StepId=self.step_id)