本文整理匯總了Python中airflow.contrib.hooks.emr_hook.EmrHook類的典型用法代碼示例。如果您正苦於以下問題：Python EmrHook類的具體用法？Python EmrHook怎麽用？Python EmrHook使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。
在下文中一共展示了EmrHook類的10個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: test_create_job_flow_uses_the_emr_config_to_create_a_cluster
def test_create_job_flow_uses_the_emr_config_to_create_a_cluster(self):
client = boto3.client('emr', region_name='us-east-1')
hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
cluster = hook.create_job_flow({'Name': 'test_cluster'})
self.assertEqual(client.list_clusters()['Clusters'][0]['Id'],
cluster['JobFlowId'])示例2: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info('Terminating JobFlow %s', self.job_flow_id)
response = emr.terminate_job_flows(JobFlowIds=[self.job_flow_id])
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('JobFlow termination failed: %s' % response)
else:
logging.info('JobFlow with id %s terminated', self.job_flow_id)示例3: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info("Adding steps to %s", self.job_flow_id)
response = emr.add_job_flow_steps(JobFlowId=self.job_flow_id, Steps=self.steps)
if not response["ResponseMetadata"]["HTTPStatusCode"] == 200:
raise AirflowException("Adding steps failed: %s" % response)
else:
logging.info("Steps %s added to JobFlow", response["StepIds"])
return response["StepIds"]示例4: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id, emr_conn_id=self.emr_conn_id)
logging.info('Creating JobFlow')
response = emr.create_job_flow(self.job_flow_overrides)
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('JobFlow creation failed: %s' % response)
else:
logging.info('JobFlow with id %s created', response['JobFlowId'])
return response['JobFlowId']示例5: execute
def execute(self, context):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
self.log.info('Adding steps to %s', self.job_flow_id)
response = emr.add_job_flow_steps(JobFlowId=self.job_flow_id, Steps=self.steps)
if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
raise AirflowException('Adding steps failed: %s' % response)
else:
self.log.info('Steps %s added to JobFlow', response['StepIds'])
return response['StepIds']示例6: test_create_job_flow_extra_args
def test_create_job_flow_extra_args(self):
"""
Test that we can add extra arguments to the launch call.
This is useful for when AWS add new options, such as
"SecurityConfiguration" so that we don't have to change our code
"""
client = boto3.client('emr', region_name='us-east-1')
hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
# AmiVersion is really old and almost no one will use it anymore, but
# it's one of the "optional" request params that moto supports - it's
# coverage of EMR isn't 100% it turns out.
cluster = hook.create_job_flow({'Name': 'test_cluster',
'ReleaseLabel': '',
'AmiVersion': '3.2'})
cluster = client.describe_cluster(ClusterId=cluster['JobFlowId'])['Cluster']
# The AmiVersion comes back as {Requested,Running}AmiVersion fields.
self.assertEqual(cluster['RequestedAmiVersion'], '3.2')示例7: test_get_conn_returns_a_boto3_connection
def test_get_conn_returns_a_boto3_connection(self):
hook = EmrHook(aws_conn_id='aws_default')
self.assertIsNotNone(hook.get_conn().list_clusters())示例8: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
logging.info('Poking cluster %s' % self.job_flow_id)
return emr.describe_cluster(ClusterId=self.job_flow_id)示例9: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
self.log.info('Poking step %s on cluster %s', self.step_id, self.job_flow_id)
return emr.describe_step(ClusterId=self.job_flow_id, StepId=self.step_id)示例10: get_emr_response
def get_emr_response(self):
emr = EmrHook(aws_conn_id=self.aws_conn_id).get_conn()
_log.info('Poking step {0} on cluster {1}'.format(self.step_id, self.job_flow_id))
return emr.describe_step(ClusterId=self.job_flow_id, StepId=self.step_id)