本文整理汇总了Python中dart.client.python.dart_client.Dart.get_engines方法的典型用法代码示例。如果您正苦于以下问题：Python Dart.get_engines方法的具体用法？Python Dart.get_engines怎么用？Python Dart.get_engines使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dart.client.python.dart_client.Dart的用法示例。
在下文中一共展示了Dart.get_engines方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: add_s3_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_s3_engine(config):
engine_config = config['engines']['s3_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving s3 engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 's3_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-s3_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-s3_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_s3'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='s3_engine',
description='For S3 operations',
options_json_schema={},
supported_action_types=[
S3ActionTypes.copy,
S3ActionTypes.data_check,
],
ecs_task_definition=ecs_task_definition
)))
_logger.info('Saved s3_engine: %s' % e1.id)示例2: add_no_op_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_no_op_engine(config):
engine_config = config['engines']['no_op_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving no_op_engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'no_op_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-no_op_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-no_op_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_no_op'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='no_op_engine',
description='Helps engineering test dart',
options_json_schema={
'type': 'object',
'properties': {
'action_sleep_time_in_seconds': {
'type': 'integer',
'minimum': 0,
'default': 5,
'description': 'The time to sleep for each action before completing'
},
},
'additionalProperties': False,
'required': [],
},
supported_action_types=[
NoOpActionTypes.action_that_succeeds,
NoOpActionTypes.action_that_fails,
NoOpActionTypes.copy_hdfs_to_s3_action,
NoOpActionTypes.load_dataset,
NoOpActionTypes.consume_subscription
],
ecs_task_definition=ecs_task_definition
)))
_logger.info('saved no_op_engine: %s' % e1.id)示例3: add_elasticsearch_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_elasticsearch_engine(config):
engine_config = config['engines']['elasticsearch_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving elasticsearch_engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'elasticsearch_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-elasticsearch_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-elasticsearch_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_elasticsearch'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='elasticsearch_engine',
description='For Elasticsearch clusters',
options_json_schema={
'type': 'object',
'properties': {
'access_key_id': {
'type': 'string',
'default': '',
'minLength': 0,
'maxLength': 20,
'description': 'the access_key_id for accessing this elasticsearch cluster. '
+ 'Leave blank to use Dart\'s instance profile credentials'
},
'secret_access_key': {
'type': 'string',
'default': '',
'minLength': 0,
'maxLength': 40,
'x-dart-secret': True,
'description': 'the secret_access_key for accessing this elasticsearch cluster. '
+ 'Leave blank to use Dart\'s instance profile credentials'
},
'endpoint': {
'type': 'string',
'minLength': 1,
'maxLength': 256,
'pattern': '^[a-zA-Z0-9]+[a-zA-Z0-9\-\.]*\.es\.amazonaws\.com$',
'description': 'The AWS Elasticsearch domain endpoint that you use to submit index and search requests.'
},
},
'additionalProperties': False,
'required': ['endpoint']
},
supported_action_types=[
ElasticsearchActionTypes.data_check,
ElasticsearchActionTypes.create_index,
ElasticsearchActionTypes.create_mapping,
ElasticsearchActionTypes.create_template,
ElasticsearchActionTypes.delete_index,
ElasticsearchActionTypes.delete_template,
ElasticsearchActionTypes.force_merge_index,
],
ecs_task_definition=ecs_task_definition
)))
_logger.info('saved elasticsearch_engine: %s' % e1.id)示例4: add_emr_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_emr_engine(config):
engine_config = config['engines']['emr_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving emr_engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'emr_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-emr_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-emr_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_emr'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='emr_engine',
description='For EMR clusters that use Hive, Impala, Spark, etc.',
options_json_schema={
'type': 'object',
'properties': {
'release_label': {'type': 'string', 'pattern': '^emr-[0-9].[0-9].[0-9]+$', 'default': 'emr-4.2.0', 'description': 'desired EMR release label'},
'instance_type': {'readonly': True, 'type': ['string', 'null'], 'default': 'm3.2xlarge', 'description': 'The ec2 instance type of master/core nodes'},
'instance_count': {'type': ['integer', 'null'], 'default': None, 'minimum': 1, 'maximum': 50, 'description': 'The total number of nodes in this cluster (overrides data_to_freespace_ratio)'},
'data_to_freespace_ratio': {'type': ['number', 'null'], 'default': 0.5, 'minimum': 0.0, 'maximum': 1.0, 'description': 'Desired ratio of HDFS data/free-space'},
'dry_run': {'type': ['boolean', 'null'], 'default': False, 'description': 'write extra_data to actions, but do not actually run'},
'ec2_keyname': {'type': 'string', 'description': 'The name of the ec2_key_pair for the emr cluster. If this is not defined, the default key-pair from config is chosen.', 'default': None},
},
'additionalProperties': False,
'required': ['release_label'],
},
supported_action_types=[
EmrActionTypes.start_datastore,
EmrActionTypes.terminate_datastore,
EmrActionTypes.load_dataset,
EmrActionTypes.consume_subscription,
EmrActionTypes.run_hive_script_action,
EmrActionTypes.run_impala_script_action,
EmrActionTypes.run_pyspark_script_action,
EmrActionTypes.copy_hdfs_to_s3_action
],
ecs_task_definition=ecs_task_definition
)))
_logger.info('saved emr_engine: %s' % e1.id)示例5: add_emr_engine_sub_graphs
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_emr_engine_sub_graphs(config):
engine_config = config['engines']['emr_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving emr_engine sub_graphs')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'emr_engine':
engine_id = e.id
if not engine_id:
raise
subgraph_definitions = [
SubGraphDefinition(data=SubGraphDefinitionData(
name='consume_subscription_workflow',
description='Add to a datastore to create entities for loading a dataset on an ongoing basis',
engine_name='emr_engine',
related_type=EntityType.datastore,
related_is_a=Relationship.PARENT,
workflows=[
Workflow(id=Ref.workflow(1), data=WorkflowData(
name='emr-workflow-consume_subscription',
datastore_id=Ref.parent(),
engine_name='emr_engine',
)),
],
subscriptions=[
Subscription(id=Ref.subscription(1), data=SubscriptionData(
name='emr-subscription',
dataset_id=''
)),
],
triggers=[
Trigger(id=Ref.trigger(1), data=TriggerData(
name='emr-trigger-subscription-1G-batch',
trigger_type_name=subscription_batch_trigger.name,
workflow_ids=[Ref.workflow(1)],
args={
'subscription_id': Ref.subscription(1),
'unconsumed_data_size_in_bytes': 1000*1000*1000
}
)),
],
actions=[
Action(id=Ref.action(1), data=ActionData(
name='emr-action-consume_subscription',
action_type_name=EmrActionTypes.consume_subscription.name,
engine_name='emr_engine',
workflow_id=Ref.workflow(1),
state=ActionState.TEMPLATE,
args={'subscription_id': Ref.subscription(1)}
)),
]
))
]
for e in subgraph_definitions:
s = dart.save_subgraph_definition(e, engine_id)
_logger.info('created subgraph_definition: %s' % s.id)示例6: add_redshift_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_redshift_engine(config):
engine_config = config['engines']['redshift_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving redshift_engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'redshift_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-redshift_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-redshift_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_redshift'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='redshift_engine',
description='For Redshift clusters',
options_json_schema={
'type': 'object',
'properties': {
'node_type': {
'type': 'string',
'default': 'ds2.xlarge',
'enum': ['ds1.xlarge', 'ds1.8xlarge', 'ds2.xlarge', 'ds2.8xlarge', 'dc1.large', 'dc1.8xlarge'],
'description': 'the type of each node'
},
'nodes': {
'type': 'integer',
'default': 2,
'minimum': 2,
'maximum': 10,
'description': 'the number of nodes in this cluster'
},
'master_user_name': {
'type': ['string', 'null'],
'default': 'admin',
'minLength': 1,
'maxLength': 128,
'pattern': '^[a-zA-Z]+[a-zA-Z0-9]*$',
'description': 'the master user name for this redshift cluster'
},
'master_user_password': {
'type': 'string',
'default': 'passw0rD--CHANGE-ME!',
'minLength': 8,
'maxLength': 64,
'pattern': '(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?!.*[\'"\/@\s])',
'x-dart-secret': True,
'description': 'the master user password for this redshift cluster (hidden and ignored after'
+ ' initial save), see AWS docs for password requirements'
},
'master_db_name': {
'type': ['string', 'null'],
"default": 'dart',
'minLength': 1,
'maxLength': 64,
'pattern': '^[a-z]+$',
'description': 'the master database name for this redshift cluster'
},
'cluster_identifier': {
'type': ['string', 'null'],
'default': None,
'minLength': 1,
'maxLength': 63,
'pattern': '^[a-zA-Z0-9-]*$',
'description': 'this overrides the auto-generated dart cluster_identifier'
},
'preferred_maintenance_window': {
'type': 'string',
'default': 'sat:03:30-sat:04:00',
#.........这里部分代码省略.........
示例7: add_no_op_engine_sub_graphs
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_no_op_engine_sub_graphs(config):
engine_config = config['engines']['no_op_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving no_op_engine sub_graphs')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'no_op_engine':
engine_id = e.id
if not engine_id:
raise
subgraph_definitions = [
SubGraphDefinition(data=SubGraphDefinitionData(
name='workflow chaining demo',
description='demonstrate workflow chaining',
engine_name='no_op_engine',
related_type=EntityType.datastore,
related_is_a=Relationship.PARENT,
workflows=[
Workflow(id=Ref.workflow(1), data=WorkflowData(
name='no-op-workflow-chaining-wf1',
datastore_id=Ref.parent(),
engine_name='no_op_engine',
state=WorkflowState.ACTIVE,
)),
Workflow(id=Ref.workflow(2), data=WorkflowData(
name='no-op-workflow-chaining-wf2',
datastore_id=Ref.parent(),
engine_name='no_op_engine',
state=WorkflowState.ACTIVE,
)),
],
actions=[
Action(id=Ref.action(1), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
action_type_name=NoOpActionTypes.action_that_succeeds.name,
workflow_id=Ref.workflow(1),
order_idx=1,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(2), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
action_type_name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(1),
order_idx=2,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(3), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
action_type_name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(1),
order_idx=3,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(4), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
action_type_name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(1),
order_idx=4,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(5), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
action_type_name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(2),
order_idx=1,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(6), data=ActionData(
name=NoOpActionTypes.action_that_succeeds.name,
action_type_name=NoOpActionTypes.action_that_succeeds.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(2),
order_idx=2,
state=ActionState.TEMPLATE,
)),
Action(id=Ref.action(7), data=ActionData(
name=NoOpActionTypes.action_that_fails.name,
action_type_name=NoOpActionTypes.action_that_fails.name,
engine_name='no_op_engine',
workflow_id=Ref.workflow(2),
order_idx=3,
state=ActionState.TEMPLATE,
)),
],
triggers=[
Trigger(id=Ref.trigger(1), data=TriggerData(
name='no-op-trigger-workflow-completion',
trigger_type_name=workflow_completion_trigger.name,
workflow_ids=[Ref.workflow(2)],
state=TriggerState.ACTIVE,
#.........这里部分代码省略.........
示例8: add_dynamodb_engine
# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import get_engines [as 别名]
def add_dynamodb_engine(config):
engine_config = config['engines']['dynamodb_engine']
opts = engine_config['options']
dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version'])
assert isinstance(dart, Dart)
_logger.info('saving dynamodb_engine')
engine_id = None
for e in dart.get_engines():
if e.data.name == 'dynamodb_engine':
engine_id = e.id
ecs_task_definition = None if config['dart']['use_local_engines'] else {
'family': 'dart-%s-dynamodb_engine' % config['dart']['env_name'],
'containerDefinitions': [
{
'name': 'dart-dynamodb_engine',
'cpu': 64,
'memory': 256,
'image': engine_config['docker_image'],
'logConfiguration': {'logDriver': 'syslog'},
'environment': [
{'name': 'DART_ROLE', 'value': 'worker:engine_dynamodb'},
{'name': 'DART_CONFIG', 'value': engine_config['config']},
{'name': 'AWS_DEFAULT_REGION', 'value': opts['emr_region']}
],
'mountPoints': [
{
'containerPath': '/mnt/ecs_agent_data',
'sourceVolume': 'ecs-agent-data',
'readOnly': True
}
],
}
],
'volumes': [
{
'host': {'sourcePath': '/var/lib/ecs/data'},
'name': 'ecs-agent-data'
}
],
}
e1 = dart.save_engine(Engine(id=engine_id, data=EngineData(
name='dynamodb_engine',
description='For DynamoDB tables',
options_json_schema={
'type': 'object',
'properties': {
'dataset_id': {'type': 'string', 'description': 'The id of the dataset on which the table is based'},
'target_table_name': {'type': ['string', 'null'], 'default': None, 'pattern': '^[a-zA-Z0-9_]+$', 'description': 'overrides dataset setting'},
'target_distribution_key': {'type': ['string', 'null'], 'default': None, 'pattern': '^[a-zA-Z0-9_]+$', 'description': 'overrides dataset setting'},
'target_sort_key': {'type': ['string', 'null'], 'default': None, 'pattern': '^[a-zA-Z0-9_]+$', 'description': 'overrides dataset setting'},
},
'additionalProperties': False,
'required': ['dataset_id'],
},
supported_action_types=[
DynamoDBActionTypes.create_table,
DynamoDBActionTypes.delete_table,
DynamoDBActionTypes.load_dataset,
],
ecs_task_definition=ecs_task_definition
)))
_logger.info('saved dynamodb_engine: %s' % e1.id)