Python Dart.save_actions方法代码示例

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
    workflow = dart.save_workflow(Workflow(
        data=WorkflowData(
            'load_beacon_native_app_impala',
            datastore.id,
            state=WorkflowState.ACTIVE,
            on_failure_email=['[email protected]'],
            on_success_email=['[email protected]'],
            on_started_email=['[email protected]'],
        )
    ), datastore.id)
    print 'created workflow: %s' % workflow.id

    a0, a1 = dart.save_actions([
        Action(data=ActionData('start_datastore', 'start_datastore', state=ActionState.TEMPLATE)),
        Action(data=ActionData('load_dataset', 'load_dataset', state=ActionState.TEMPLATE, args={
            'dataset_id': dataset.id,
            's3_path_start_prefix_inclusive': 's3://example-bucket/prd/beacon/native_app/v2/parquet/snappy/createdpartition=2015-06-27',
        })),
    ], workflow_id=workflow.id)
    print 'created action: %s' % a0.id
    print 'created action: %s' % a1.id

    event = dart.save_event(Event(data=EventData('beacon_native_app_to_parquet_emr_job_completion', state=EventState.ACTIVE)))
    print 'created event: %s' % event.id

    trigger = dart.save_trigger(Trigger(data=TriggerData(
        'beacon_native_app_to_parquet_emr_job_completion_trigger',
        'event',
        [workflow.id],
        {'event_id': event.id}))
    )

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
            name='weblogs_DW-3503',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                'data_to_freespace_ratio': 0.30,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': dataset.id,
                's3_path_start_prefix_inclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/50',
                's3_path_end_prefix_exclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2015/00',
                's3_path_regex_filter': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/../www\\.retailmenot\\.com.*',
                'target_file_format': FileFormat.TEXTFILE,
                'target_row_format': RowFormat.DELIMITED,
                'target_compression': Compression.GZIP,
                'target_delimited_by': '\t',
                'target_quoted_by': '"',
                'target_escaped_by': '\\',
                'target_null_string': 'NULL',
            })),
        ],
        datastore_id=datastore.id
    )
    print 'created datastore action: %s' % actions[0].id

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
class TestActionCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData(name='test-datastore', engine_name='no_op_engine', args=args, state=DatastoreState.TEMPLATE))
        self.datastore = self.dart.save_datastore(dst)
        wf = Workflow(data=WorkflowData(name='test-workflow', datastore_id=self.datastore.id))
        self.workflow = self.dart.save_workflow(workflow=wf, datastore_id=self.datastore.id)
        self.maxDiff = 99999

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)
        self.dart.delete_workflow(self.workflow.id)

    def test_crud_datastore(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions(actions=[action0, action1], datastore_id=self.datastore.id)

        # copy fields that are populated at creation time
        action0.data.datastore_id = posted_actions[0].data.datastore_id
        action1.data.datastore_id = posted_actions[1].data.datastore_id
        action0.data.args = {}
        action1.data.args = {}
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        # When retrieving an action, its queue time and state
        # differs from the action default values created by action0 and action1
        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        action0.data.state = a0.data.state
        action1.data.state = a1.data.state
        action0.data.queued_time = a0.data.queued_time
        action1.data.queued_time = a1.data.queued_time

        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return

        self.fail('action should have been missing after delete!')

    def test_crud_workflow(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions([action0, action1], workflow_id=self.workflow.id)

        # copy fields that are populated at creation time
        action0.data.workflow_id = posted_actions[0].data.workflow_id
        action1.data.workflow_id = posted_actions[1].data.workflow_id
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx
        action0.data.args = {}
        action1.data.args = {}

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return
#.........这里部分代码省略.........

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
            state=DatastoreState.ACTIVE,
            args={
                # 'data_to_freespace_ratio': 0.10,
                'instance_count': 2,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': dataset.id,
                's3_path_end_prefix_exclusive': 's3://example-bucket/prd/inbound/overlord/raw/rmndirect/2015/09/04/',
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,
            })),
        ],
        datastore_id=datastore.id
    )
    print 'created action: %s' % actions[0].id
    print 'created action: %s' % actions[1].id

    workflow = dart.save_workflow(
        workflow=Workflow(
            data=WorkflowData(
                name='rmn_direct_workflow_DW-3307',
                datastore_id=datastore.id,
                state=WorkflowState.ACTIVE,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='amaceiras_beacon_native_app_null_coupons_issue',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                # 'data_to_freespace_ratio': 0.05,
                'instance_count': 5,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': 'URBA9XEQEF',
                's3_path_start_prefix_inclusive': 's3://example-bucket/nb.retailmenot.com/parsed_logs/2015/33/beacon-v2-2015-08-18',
                # 's3_path_end_prefix_exclusive': 's3://example-bucket/nb.retailmenot.com/parsed_logs/2015/31/beacon-v2-2015-08-01',
                's3_path_regex_filter': '.*\\.tsv',
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,
            })),
        ],
        datastore_id=datastore.id
        # datastore_id='JK4PVO2KZA'
    )
    print 'created action: %s' % actions[0].id

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
from dart.client.python.dart_client import Dart
from dart.model.action import Action
from dart.model.action import ActionData
from dart.model.dataset import FileFormat

if __name__ == '__main__':
    dart = Dart('localhost', 5000)
    assert isinstance(dart, Dart)

    action = dart.save_actions([
        Action(data=ActionData('load_dataset', 'load_dataset', args={
            'dataset_id': 'NVVLBI7CWB',
            's3_path_start_prefix_inclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/52',
            's3_path_end_prefix_exclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2015/00',
            's3_path_regex_filter': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/../www\\.retailmenot\\.com.*',
            'target_file_format': FileFormat.PARQUET,
        })),
    ], datastore_id='IOMUQ5L8AX')[0]
    print 'created action: %s' % action.id

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_actions [as 别名]
                on_success_email=['[email protected]', '[email protected]'],
                on_started_email=['[email protected]', '[email protected]'],
            )
        ),
        datastore_id=datastore.id
    )
    print 'created workflow: %s' % workflow.id

    a2 = dart.save_actions(
        actions=[
            Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={
                'subscription_id': subscription.id,
                'target_file_format': FileFormat.TEXTFILE,
                'target_row_format': RowFormat.DELIMITED,
                'target_compression': Compression.GZIP,
                'target_delimited_by': '\t',
                'target_quoted_by': '"',
                'target_escaped_by': '\\',
                'target_null_string': 'NULL',
            })),
        ],
        workflow_id=workflow.id
    )[0]
    print 'created workflow action: %s' % a2.id

    trigger = dart.save_trigger(Trigger(data=TriggerData(
        name='weblogs_DW-3213_v3',
        trigger_type_name='subscription_batch',
        workflow_ids=[workflow.id],
        args={
            'subscription_id': subscription.id,