当前位置: 首页>>代码示例>>Python>>正文


Python ProcessContext.get_sink方法代码示例

本文整理汇总了Python中system.process_context.ProcessContext.get_sink方法的典型用法代码示例。如果您正苦于以下问题:Python ProcessContext.get_sink方法的具体用法?Python ProcessContext.get_sink怎么用?Python ProcessContext.get_sink使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在system.process_context.ProcessContext的用法示例。


在下文中一共展示了ProcessContext.get_sink方法的4个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: _start_process

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_sink [as 别名]
    def _start_process(self, start_timeperiod, end_timeperiod, arguments):
        try:
            start_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, start_timeperiod)
            sqoop_slice_starttime = start_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)

            end_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, end_timeperiod)
            sqoop_slice_endtime = end_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)

            sink_path = ProcessContext.get_sink(self.process_name)

            self.logger.info('start: %s {' % self.process_name)
            p = psutil.Popen([settings['bash_shell'],
                              settings['sqoop_command'],
                              str(sqoop_slice_starttime),
                              str(sqoop_slice_endtime),
                              sink_path + '/' + start_timeperiod],
                             close_fds=True,
                             cwd=settings['process_cwd'],
                             stdin=PIPE,
                             stdout=PIPE,
                             stderr=PIPE)
            self.cli_process = p
            self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True)
        finally:
            self.logger.info('}')
开发者ID:NMerch,项目名称:scheduler,代码行数:29,代码来源:sqoop_driver.py

示例2: create_unit_of_work

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_sink [as 别名]
def create_unit_of_work(process_name,
                        start_id,
                        end_id,
                        timeperiod='INVALID_TIMEPERIOD',
                        state=unit_of_work.STATE_REQUESTED,
                        creation_at=datetime.utcnow(),
                        uow_id=None):
    """ method creates and returns unit_of_work """
    try:
        source_collection = ProcessContext.get_source(process_name)
        target_collection = ProcessContext.get_sink(process_name)
    except KeyError:
        source_collection = None
        target_collection = None

    uow = UnitOfWork()
    uow.timeperiod = timeperiod
    uow.start_timeperiod = timeperiod
    uow.end_timeperiod = timeperiod
    uow.start_id = start_id
    uow.end_id = end_id
    uow.source = source_collection
    uow.sink = target_collection
    uow.state = state
    uow.created_at = creation_at
    uow.process_name = process_name
    uow.number_of_retries = 0

    if uow_id is not None:
        uow.document['_id'] = uow_id

    return uow
开发者ID:NMerch,项目名称:scheduler,代码行数:34,代码来源:base_fixtures.py

示例3: compute_scope_of_processing

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_sink [as 别名]
    def compute_scope_of_processing(self, process_name, start_timeperiod, end_timeperiod, job_record):
        """method reads collection and identify slice for processing"""
        source_collection_name = ProcessContext.get_source(process_name)
        target_collection_name = ProcessContext.get_sink(process_name)

        start_id = self.ds.highest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)
        end_id = self.ds.lowest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)

        uow = UnitOfWork()
        uow.timeperiod = start_timeperiod
        uow.start_id = str(start_id)
        uow.end_id = str(end_id)
        uow.start_timeperiod = start_timeperiod
        uow.end_timeperiod = end_timeperiod
        uow.created_at = datetime.utcnow()
        uow.source = source_collection_name
        uow.sink = target_collection_name
        uow.state = unit_of_work.STATE_REQUESTED
        uow.process_name = process_name
        uow.number_of_retries = 0
        uow_id = self.uow_dao.insert(uow)

        mq_request = WorkerMqRequest()
        mq_request.process_name = process_name
        mq_request.unit_of_work_id = uow_id

        publisher = self.publishers.get(process_name)
        publisher.publish(mq_request.document)
        publisher.release()

        msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_timeperiod)
        self._log_message(INFO, process_name, job_record, msg)
        return uow
开发者ID:NMerch,项目名称:scheduler,代码行数:35,代码来源:continuous_pipeline.py

示例4: __init__

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_sink [as 别名]
    def __init__(self, process_name):
        """@param process_name: id of the process, the worker will be performing """
        super(AbstractMqWorker, self).__init__(process_name)
        self.queue_source = ProcessContext.get_source(self.process_name)
        self.queue_sink = ProcessContext.get_sink(self.process_name)
        self.consumer = None
        self._init_mq_consumer()

        self.main_thread = None
        self.performance_ticker = None
        self._init_performance_ticker(self.logger)

        msg_suffix = 'in Production Mode'
        if settings['under_test']:
            msg_suffix = 'in Testing Mode'
        self.logger.info('Started %s %s' % (self.process_name, msg_suffix))
开发者ID:NMerch,项目名称:scheduler,代码行数:18,代码来源:abstract_mq_worker.py


注:本文中的system.process_context.ProcessContext.get_sink方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。