Python Dart.save_datastore方法代码示例

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
class TestWorkflowCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData('test-datastore', 'no_op_engine', args=args, state=DatastoreState.ACTIVE))
        self.datastore = self.dart.save_datastore(dst)

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)

    def test_crud(self):
        wf = Workflow(data=WorkflowData('test-workflow', self.datastore.id, engine_name='no_op_engine'))
        posted_wf = self.dart.save_workflow(wf, self.datastore.id)
        self.assertEqual(posted_wf.data.to_dict(), wf.data.to_dict())

        workflow = self.dart.get_workflow(posted_wf.id)
        self.assertEqual(posted_wf.to_dict(), workflow.to_dict())

        workflow.data.concurrency = 2
        workflow.data.state = WorkflowState.ACTIVE
        put_workflow = self.dart.save_workflow(workflow)
        # not all properties can be modified
        self.assertEqual(put_workflow.data.concurrency, 1)
        self.assertEqual(put_workflow.data.state, WorkflowState.ACTIVE)
        self.assertNotEqual(posted_wf.to_dict(), put_workflow.to_dict())

        self.dart.delete_workflow(workflow.id)
        try:
            self.dart.get_workflow(workflow.id)
        except DartRequestException as e:
            self.assertEqual(e.response.status_code, 404)
            return

        self.fail('workflow should have been missing after delete!')

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
class TestTriggerCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData('test-datastore', 'no_op_engine', args=args, state=DatastoreState.TEMPLATE))
        self.datastore = self.dart.save_datastore(dst)
        wf = Workflow(data=WorkflowData('test-workflow', self.datastore.id, state=WorkflowState.ACTIVE))
        self.workflow = self.dart.save_workflow(wf, self.datastore.id)

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)
        self.dart.delete_workflow(self.workflow.id)

    def test_crud(self):
        args = {'completed_workflow_id': self.workflow.id}
        tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', [self.workflow.id], args))
        posted_tr = self.dart.save_trigger(tr)
        self.assertEqual(posted_tr.data.to_dict(), tr.data.to_dict())

        trigger = self.dart.get_trigger(posted_tr.id)
        self.assertEqual(posted_tr.to_dict(), trigger.to_dict())

        self.dart.delete_trigger(trigger.id)
        try:
            self.dart.get_trigger(trigger.id)
        except DartRequestException as e:
            self.assertEqual(e.response.status_code, 404)
            return

        self.fail('trigger should have been missing after delete!')

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
class TestDatastoreCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)

    def test_crud(self):
        dst = Datastore(data=DatastoreData(
            name='test-datastore',
            engine_name='no_op_engine',
            args={'action_sleep_time_in_seconds': 0},
            tags=['foo']
        ))
        posted_datastore = self.dart.save_datastore(dst)

        # copy fields that are populated at creation time
        dst.data.s3_artifacts_path = posted_datastore.data.s3_artifacts_path
        dst.data.s3_logs_path = posted_datastore.data.s3_logs_path
        dst.data.user_id = posted_datastore.data.user_id
        self.assertEqual(posted_datastore.data.to_dict(), dst.data.to_dict())

        datastore = self.dart.get_datastore(posted_datastore.id)
        self.assertEqual(posted_datastore.to_dict(), datastore.to_dict())

        datastore.data.engine_name = 'not_existing_engine'
        datastore.data.state = DatastoreState.ACTIVE
        put_datastore = self.dart.save_datastore(datastore)
        # not all properties can be modified
        self.assertEqual(put_datastore.data.engine_name, 'no_op_engine')
        self.assertEqual(put_datastore.data.state, DatastoreState.ACTIVE)
        self.assertNotEqual(posted_datastore.to_dict(), put_datastore.to_dict())

        self.dart.delete_datastore(datastore.id)
        try:
            self.dart.get_datastore(datastore.id)
        except DartRequestException as e:
            self.assertEqual(e.response.status_code, 404)
            return

        self.fail('datastore should have been missing after delete!')

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
            Column('fruitSlice', DataType.STRING),
            Column('cacheHitMiss', DataType.STRING),
        ],
        compression=Compression.BZ2,
        partitions=[
            Column('year', DataType.STRING),
            Column('week', DataType.STRING),
        ],
    )))
    print 'created dataset: %s' % dataset.id

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='weblogs_DW-3503',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                'data_to_freespace_ratio': 0.30,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': dataset.id,
                's3_path_start_prefix_inclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/50',
                's3_path_end_prefix_exclusive': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2015/00',
                's3_path_regex_filter': 's3://example-bucket/weblogs/www.retailmenot.com/ec2/2014/../www\\.retailmenot\\.com.*',
                'target_file_format': FileFormat.TEXTFILE,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
            Column('featuredCouponPosition', DataType.INT),
            Column('commentCount', DataType.INT),
            Column('mallCount', DataType.INT),
            Column('clickCount', DataType.INT),
            Column('merchantName', DataType.STRING),
            Column('merchantPosition', DataType.INT),
        ],
        compression=Compression.SNAPPY,
        partitions=[Column('createdpartition', DataType.STRING)],
    ))))
    print 'created dataset: %s' % dataset.id

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            'beacon_native_app_impala',
            'emr_engine',
            state=DatastoreState.TEMPLATE,
            args={'data_to_freespace_ratio': 0.25}
        )
    ))
    print 'created datastore: %s' % datastore.id

    workflow = dart.save_workflow(Workflow(
        data=WorkflowData(
            'load_beacon_native_app_impala',
            datastore.id,
            state=WorkflowState.ACTIVE,
            on_failure_email=['[email protected]'],
            on_success_email=['[email protected]'],
            on_started_email=['[email protected]'],
        )
    ), datastore.id)

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
class TestActionCrud(unittest.TestCase):
    def setUp(self):
        self.dart = Dart(host='localhost', port=5000)
        args = {'action_sleep_time_in_seconds': 0}
        dst = Datastore(data=DatastoreData(name='test-datastore', engine_name='no_op_engine', args=args, state=DatastoreState.TEMPLATE))
        self.datastore = self.dart.save_datastore(dst)
        wf = Workflow(data=WorkflowData(name='test-workflow', datastore_id=self.datastore.id))
        self.workflow = self.dart.save_workflow(workflow=wf, datastore_id=self.datastore.id)
        self.maxDiff = 99999

    def tearDown(self):
        self.dart.delete_datastore(self.datastore.id)
        self.dart.delete_workflow(self.workflow.id)

    def test_crud_datastore(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name,
                                         action_type_name=NoOpActionTypes.action_that_succeeds.name,
                                         engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions(actions=[action0, action1], datastore_id=self.datastore.id)

        # copy fields that are populated at creation time
        action0.data.datastore_id = posted_actions[0].data.datastore_id
        action1.data.datastore_id = posted_actions[1].data.datastore_id
        action0.data.args = {}
        action1.data.args = {}
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        # When retrieving an action, its queue time and state
        # differs from the action default values created by action0 and action1
        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        action0.data.state = a0.data.state
        action1.data.state = a1.data.state
        action0.data.queued_time = a0.data.queued_time
        action1.data.queued_time = a1.data.queued_time

        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return

        self.fail('action should have been missing after delete!')

    def test_crud_workflow(self):
        action0 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        action1 = Action(data=ActionData(name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE, engine_name='no_op_engine'))
        posted_actions = self.dart.save_actions([action0, action1], workflow_id=self.workflow.id)

        # copy fields that are populated at creation time
        action0.data.workflow_id = posted_actions[0].data.workflow_id
        action1.data.workflow_id = posted_actions[1].data.workflow_id
        action0.data.order_idx = posted_actions[0].data.order_idx
        action1.data.order_idx = posted_actions[1].data.order_idx
        action0.data.args = {}
        action1.data.args = {}

        action0.data.user_id = posted_actions[0].data.user_id
        action1.data.user_id = posted_actions[1].data.user_id

        self.assertEqual(posted_actions[0].data.to_dict(), action0.data.to_dict())
        self.assertEqual(posted_actions[1].data.to_dict(), action1.data.to_dict())

        a0 = self.dart.get_action(posted_actions[0].id)
        a1 = self.dart.get_action(posted_actions[1].id)
        self.assertEqual(a0.data.to_dict(), action0.data.to_dict())
        self.assertEqual(a1.data.to_dict(), action1.data.to_dict())

        self.dart.delete_action(a0.id)
        self.dart.delete_action(a1.id)

        try:
            self.dart.get_action(a0.id)
        except DartRequestException as e0:
            self.assertEqual(e0.response.status_code, 404)
            try:
                self.dart.get_action(a1.id)
            except DartRequestException as e1:
                self.assertEqual(e1.response.status_code, 404)
                return
#.........这里部分代码省略.........

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
        on_success_email=['[email protected]'],
        on_failure_email=['[email protected]'],
    )))
    print 'created subscription: %s' % subscription.id

    print 'awaiting subscription generation...'
    subscription = dart.await_subscription_generation(subscription.id)
    assert subscription.data.state == SubscriptionState.ACTIVE
    print 'done.'

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='rmn_direct_adhoc_DW-3307',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                # 'data_to_freespace_ratio': 0.10,
                'instance_count': 2,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': dataset.id,
                's3_path_end_prefix_exclusive': 's3://example-bucket/prd/inbound/overlord/raw/rmndirect/2015/09/04/',
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
from dart.client.python.dart_client import Dart
from dart.model.datastore import Datastore, DatastoreState

if __name__ == '__main__':
    dart = Dart('localhost', 5000)
    assert isinstance(dart, Dart)

    datastore = dart.get_datastore('KNMUGQWTHT')
    assert isinstance(datastore, Datastore)

    datastore.data.state = DatastoreState.ACTIVE
    dart.save_datastore(datastore)

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
        on_success_email=['[email protected]'],
        on_failure_email=['[email protected]'],
    )))
    print 'created subscription: %s' % subscription.id

    print 'awaiting subscription generation...'
    subscription = dart.await_subscription_generation(subscription.id)
    assert subscription.data.state == SubscriptionState.ACTIVE
    print 'done.'

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='owen_eu_parquet_DW-3213_v3',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                # 'data_to_freespace_ratio': 0.05,
                'instance_count': 3,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    a0, a1 = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': dataset.id,
                's3_path_end_prefix_exclusive': 's3://example-bucket/prd/inbound/overlord/eu-all-events/2015/08/05/',
                'target_file_format': FileFormat.PARQUET,
                'target_row_format': RowFormat.NONE,

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
from dart.model.dataset import FileFormat, Compression
from dart.model.dataset import RowFormat
from dart.model.datastore import Datastore
from dart.model.datastore import DatastoreData
from dart.model.datastore import DatastoreState

if __name__ == '__main__':
    dart = Dart('localhost', 5000)
    assert isinstance(dart, Dart)

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='amaceiras_beacon_native_app_null_coupons_issue',
            engine_name='emr_engine',
            state=DatastoreState.ACTIVE,
            args={
                # 'data_to_freespace_ratio': 0.05,
                'instance_count': 5,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    actions = dart.save_actions(
        actions=[
            Action(data=ActionData('start_datastore', 'start_datastore')),
            Action(data=ActionData('load_dataset', 'load_dataset', args={
                'dataset_id': 'URBA9XEQEF',
                's3_path_start_prefix_inclusive': 's3://example-bucket/nb.retailmenot.com/parsed_logs/2015/33/beacon-v2-2015-08-18',
                # 's3_path_end_prefix_exclusive': 's3://example-bucket/nb.retailmenot.com/parsed_logs/2015/31/beacon-v2-2015-08-01',
                's3_path_regex_filter': '.*\\.tsv',

# 需要导入模块: from dart.client.python.dart_client import Dart [as 别名]
# 或者: from dart.client.python.dart_client.Dart import save_datastore [as 别名]
        dataset_id=dataset.id,
        on_failure_email=['[email protected]', '[email protected]'],
        on_success_email=['[email protected]', '[email protected]'],
    )))
    print 'created subscription: %s' % subscription.id

    print 'awaiting subscription generation...'
    subscription = dart.await_subscription_generation(subscription.id)
    assert subscription.data.state == SubscriptionState.ACTIVE
    print 'done.'

    datastore = dart.save_datastore(Datastore(
        data=DatastoreData(
            name='weblogs_rmn_legacy',
            engine_name='emr_engine',
            state=DatastoreState.TEMPLATE,
            args={
                'data_to_freespace_ratio': 0.50,
            }
        )
    ))
    print 'created datastore: %s' % datastore.id

    workflow = dart.save_workflow(
        workflow=Workflow(
            data=WorkflowData(
                name='weblogs_rmn_legacy_parse_to_delimited',
                datastore_id=datastore.id,
                state=WorkflowState.ACTIVE,
                on_failure_email=['[email protected]', '[email protected]'],
                on_success_email=['[email protected]', '[email protected]'],
                on_started_email=['[email protected]', '[email protected]'],