Python Dart.save_workflow方法代码示例

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
class TestWorkflowCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData('test-datastore', 'no_op_engine', args=args, state=DatastoreState.ACTIVE))
        self.datastore = self.dart.save_datastore(dst)

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)

    def test_crud(self):
        wf = Workflow(data=WorkflowData('test-workflow', self.datastore.id, engine_name='no_op_engine'))
        posted_wf = self.dart.save_workflow(wf, self.datastore.id)
        self.assertEqual(posted_wf.data.to_dict(), wf.data.to_dict())

        workflow = self.dart.get_workflow(posted_wf.id)
        self.assertEqual(posted_wf.to_dict(), workflow.to_dict())

        workflow.data.concurrency = 2
        workflow.data.state = WorkflowState.ACTIVE
        put_workflow = self.dart.save_workflow(workflow)
        # not all properties can be modified
        self.assertEqual(put_workflow.data.concurrency, 1)
        self.assertEqual(put_workflow.data.state, WorkflowState.ACTIVE)
        self.assertNotEqual(posted_wf.to_dict(), put_workflow.to_dict())

        self.dart.delete_workflow(workflow.id)
        try:
            self.dart.get_workflow(workflow.id)
        except DartRequestException as e:
            self.assertEqual(e.response.status_code, 404)
            return

        self.fail('workflow should have been missing after delete!')

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
class TestTriggerCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData('test-datastore', 'no_op_engine', args=args, state=DatastoreState.TEMPLATE))
        self.datastore = self.dart.save_datastore(dst)
        wf = Workflow(data=WorkflowData('test-workflow', self.datastore.id, state=WorkflowState.ACTIVE))
        self.workflow = self.dart.save_workflow(wf, self.datastore.id)

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)
        self.dart.delete_workflow(self.workflow.id)

    def test_crud(self):
        args = {'completed_workflow_id': self.workflow.id}
        tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', [self.workflow.id], args))
        posted_tr = self.dart.save_trigger(tr)
        self.assertEqual(posted_tr.data.to_dict(), tr.data.to_dict())

        trigger = self.dart.get_trigger(posted_tr.id)
        self.assertEqual(posted_tr.to_dict(), trigger.to_dict())

        self.dart.delete_trigger(trigger.id)
        try:
            self.dart.get_trigger(trigger.id)
        except DartRequestException as e:
            self.assertEqual(e.response.status_code, 404)
            return

        self.fail('trigger should have been missing after delete!')

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
from dart.client.python.dart_client import Dart
from dart.model.workflow import Workflow, WorkflowState

if __name__ == '__main__':
    dart = Dart('localhost', 5000)
    assert isinstance(dart, Dart)

    workflow = dart.get_workflow('456SGU4U6T')
    assert isinstance(workflow, Workflow)

    workflow.data.state = WorkflowState.INACTIVE
    dart.save_workflow(workflow)

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            'beacon_native_app_impala',
            'emr_engine',
            state=DatastoreState.TEMPLATE,
            args={'data_to_freespace_ratio': 0.25}
        )
    ))
    print 'created datastore: %s' % datastore.id

    workflow = dart.save_workflow(Workflow(
        data=WorkflowData(
            'load_beacon_native_app_impala',
            datastore.id,
            state=WorkflowState.ACTIVE,
            on_failure_email=['[email protected]'],
            on_success_email=['[email protected]'],
            on_started_email=['[email protected]'],
        )
    ), datastore.id)
    print 'created workflow: %s' % workflow.id

    a0, a1 = dart.save_actions([
        Action(data=ActionData('start_datastore', 'start_datastore', state=ActionState.TEMPLATE)),
        Action(data=ActionData('load_dataset', 'load_dataset', state=ActionState.TEMPLATE, args={
            'dataset_id': dataset.id,
            's3_path_start_prefix_inclusive': 's3://example-bucket/prd/beacon/native_app/v2/parquet/snappy/createdpartition=2015-06-27',
        })),
    ], workflow_id=workflow.id)
    print 'created action: %s' % a0.id
    print 'created action: %s' % a1.id

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
class TestActionCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData(name='test-datastore', engine_name='no_op_engine', args=args, state=DatastoreState.TEMPLATE))
        self.datastore = self.dart.save_datastore(dst)
        wf = Workflow(data=WorkflowData(name='test-workflow', datastore_id=self.datastore.id))
        self.workflow = self.dart.save_workflow(workflow=wf, datastore_id=self.datastore.id)
        self.maxDiff = 99999

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)
        self.dart.delete_workflow(self.workflow.id)

    def test_crud_datastore(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions(actions=[action0, action1], datastore_id=self.datastore.id)

        # copy fields that are populated at creation time
        action0.data.datastore_id = posted_actions[0].data.datastore_id
        action1.data.datastore_id = posted_actions[1].data.datastore_id
        action0.data.args = {}
        action1.data.args = {}
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        # When retrieving an action, its queue time and state
        # differs from the action default values created by action0 and action1
        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        action0.data.state = a0.data.state
        action1.data.state = a1.data.state
        action0.data.queued_time = a0.data.queued_time
        action1.data.queued_time = a1.data.queued_time

        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return

        self.fail('action should have been missing after delete!')

    def test_crud_workflow(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions([action0, action1], workflow_id=self.workflow.id)

        # copy fields that are populated at creation time
        action0.data.workflow_id = posted_actions[0].data.workflow_id
        action1.data.workflow_id = posted_actions[1].data.workflow_id
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx
        action0.data.args = {}
        action1.data.args = {}

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return
#.........这里部分代码省略.........

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,
            })),
        ],
        datastore_id=datastore.id
    )
    print 'created action: %s' % actions[0].id
    print 'created action: %s' % actions[1].id

    workflow = dart.save_workflow(
        workflow=Workflow(
            data=WorkflowData(
                name='rmn_direct_workflow_DW-3307',
                datastore_id=datastore.id,
                state=WorkflowState.ACTIVE,
                on_failure_email=['[email protected]'],
                on_success_email=['[email protected]'],
                on_started_email=['[email protected]'],
            )
        ),
        datastore_id=datastore.id
    )
    print 'created workflow: %s' % workflow.id

    wf_actions = dart.save_actions(
        actions=[
            Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={
                'subscription_id': subscription.id,
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,
            })),
        ],
        datastore_id=datastore.id
    )
    print 'created action: %s' % a0.id
    print 'created action: %s' % a1.id

    workflow = dart.save_workflow(
        workflow=Workflow(
            data=WorkflowData(
                name='owen_eu_parquet_workflow_DW-3213_v3',
                datastore_id=datastore.id,
                state=WorkflowState.ACTIVE,
                on_failure_email=['[email protected]'],
                on_success_email=['[email protected]'],
                on_started_email=['[email protected]'],
            )
        ),
        datastore_id=datastore.id
    )
    print 'created workflow: %s' % workflow.id

    a2 = dart.save_actions(
        actions=[
            Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={
                'subscription_id': subscription.id,
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,
                'target_compression': Compression.SNAPPY,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_workflow [as 别名]
            engine_name='emr_engine',
            state=DatastoreState.TEMPLATE,
            args={
                'data_to_freespace_ratio': 0.50,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    workflow = dart.save_workflow(
        workflow=Workflow(
            data=WorkflowData(
                name='weblogs_rmn_legacy_parse_to_delimited',
                datastore_id=datastore.id,
                state=WorkflowState.ACTIVE,
                on_failure_email=['[email protected]', '[email protected]'],
                on_success_email=['[email protected]', '[email protected]'],
                on_started_email=['[email protected]', '[email protected]'],
            )
        ),
        datastore_id=datastore.id
    )
    print 'created workflow: %s' % workflow.id

    a2 = dart.save_actions(
        actions=[
            Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={
                'subscription_id': subscription.id,
                'target_file_format': FileFormat.TEXTFILE,
                'target_row_format': RowFormat.DELIMITED,
                'target_compression': Compression.GZIP,