Python UnitOfWorkDao.get_one方法代码示例

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class BashRunnable(threading.Thread):
    """Process starts remote or local bash script job, supervises its execution and updates mq"""

    def __init__(self, logger, message, consumer, performance_ticker):
        self.logger = logger
        self.message = message
        self.mq_request = SynergyMqTransmission.from_json(message.body)
        self.consumer = consumer
        self.performance_ticker = performance_ticker
        self.alive = False
        self.return_code = -1
        self.uow_dao = UnitOfWorkDao(self.logger)

        self.thread_name = str(self.mq_request)
        super(BashRunnable, self).__init__(name=self.thread_name)

    def _poll_process(self):
        return self.alive, self.return_code

    def _start_process(self):
        try:
            uow = self.uow_dao.get_one(self.mq_request.unit_of_work_id)
            if not uow.is_requested:
                # accept only UOW in STATE_REQUESTED
                self.logger.warn('Skipping unit_of_work: id %s; state %s;' % (str(self.message.body), uow.state),
                                 exc_info=False)
                self.consumer.acknowledge(self.message.delivery_tag)
                return
        except Exception:
            self.logger.error('Safety fuse. Can not identify unit_of_work %s' % str(self.message.body), exc_info=True)
            self.consumer.acknowledge(self.message.delivery_tag)
            return

        try:
            self.logger.info('start: %s {' % self.thread_name)
            self.alive = True

            uow.state = unit_of_work.STATE_IN_PROGRESS
            uow.started_at = datetime.utcnow()
            self.uow_dao.update(uow)

            fabric.operations.env.warn_only = True
            fabric.operations.env.abort_on_prompts = True
            fabric.operations.env.use_ssh_config = True
            fabric.operations.env.host_string = uow.arguments[ARGUMENT_CMD_HOST]

            command = os.path.join(uow.arguments[ARGUMENT_CMD_PATH],
                                   uow.arguments[ARGUMENT_CMD_FILE])
            command += ' %s' % uow.arguments[ARGUMENT_CMD_ARGS]

            run_result = fabric.operations.run(command, pty=False)
            if run_result.succeeded:
                self.return_code = 0

            uow.finished_at = datetime.utcnow()
            uow.state = unit_of_work.STATE_PROCESSED
            self.uow_dao.update(uow)

            self.logger.info('Completed %s with result = %r' % (self.thread_name, self.return_code))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.thread_name, exc_info=True)
            uow.state = unit_of_work.STATE_INVALID
            self.uow_dao.update(uow)
        finally:
            self.logger.info('}')
            self.alive = False

    def run(self):
        try:
            self._start_process()
            code = None
            alive = True
            while alive:
                alive, code = self._poll_process()
                time.sleep(0.1)

            if code == 0:
                self.performance_ticker.tracker.increment_success()
            else:
                self.performance_ticker.tracker.increment_failure()

            self.logger.info('BashDriver for %s return code is %r' % (self.thread_name, code))
        except Exception as e:
            self.performance_ticker.tracker.increment_failure()
            self.logger.error('Safety fuse while processing request %r: %r' % (self.message.body, e), exc_info=True)
        finally:
            self.consumer.acknowledge(self.message.delivery_tag)

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class Timetable(object):
    """ Timetable holds all known process trees, where every node presents a timeperiod-driven job"""

    def __init__(self, logger):
        self.lock = RLock()
        self.logger = logger
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.job_dao = JobDao(self.logger)
        self.reprocess = dict()

        # self.trees contain all of the trees and manages much of their life cycle
        # remember to enlist there all trees the system is working with
        self.trees = self._construct_trees_from_context()

        self._register_callbacks()
        self._register_dependencies()
        self.load_tree()
        self.build_trees()
        self.validate()

    def _construct_trees_from_context(self):
        trees = dict()
        for tree_name, context_entry in context.timetable_context.items():
            tree = MultiLevelTree(process_names=context_entry.enclosed_processes,
                                  tree_name=tree_name,
                                  mx_name=context_entry.mx_name,
                                  mx_page=context_entry.mx_page)
            trees[tree_name] = tree
        return trees

    def _register_dependencies(self):
        """ register dependencies between trees"""
        for tree_name, context_entry in context.timetable_context.items():
            tree = self.trees[tree_name]
            assert isinstance(tree, MultiLevelTree)
            for dependent_on in context_entry.dependent_on:
                dependent_on_tree = self.trees[dependent_on]
                assert isinstance(dependent_on_tree, MultiLevelTree)
                tree.register_dependent_on(dependent_on_tree)

    def _register_callbacks(self):
        """ register logic that reacts on reprocessing request
        and create embryo timetable record request"""

        # reprocessing request
        for tree_name, tree in self.trees.items():
            tree.register_reprocess_callback(self._callback_reprocess)

        # skip request
        for tree_name, tree in self.trees.items():
            tree.register_skip_callback(self._callback_skip)

        # callbacks register
        for tree_name, tree in self.trees.items():
            tree.register_create_callbacks(self._callback_create_job_record)

    # *** Call-back methods ***
    def _find_dependant_trees(self, tree_obj):
        """ returns list of trees that are dependent_on given tree_obj """
        dependant_trees = []
        for tree_name, tree in self.trees.items():
            if tree_obj in tree.dependent_on:
                dependant_trees.append(tree)
        return dependant_trees

    def _find_dependant_tree_nodes(self, node_a):
        dependant_nodes = set()
        for tree_b in self._find_dependant_trees(node_a.tree):
            node_b = node_a.find_counterpart_in(tree_b)
            if node_b is None:
                continue
            dependant_nodes.add(node_b)
        return dependant_nodes

    def _reprocess_single_tree_node(self, tree_node):
        """ is called from tree to answer reprocessing request.
        It is possible that timetable record will be transferred to STATE_IN_PROGRESS with no related unit_of_work"""
        uow_id = tree_node.job_record.related_unit_of_work
        if uow_id is not None:
            tree_node.job_record.state = job.STATE_IN_PROGRESS
            uow = self.uow_dao.get_one(uow_id)
            uow.state = unit_of_work.STATE_INVALID
            uow.number_of_retries = 0
            uow.created_at = datetime.utcnow()
            self.uow_dao.update(uow)
            msg = 'Transferred job record %s for %s in timeperiod %s to %s; Transferred unit_of_work to %s' \
                  % (tree_node.job_record.db_id,
                     tree_node.process_name,
                     tree_node.job_record.timeperiod,
                     tree_node.job_record.state,
                     uow.state)

            if tree_node.process_name not in self.reprocess:
                self.reprocess[tree_node.process_name] = dict()
            self.reprocess[tree_node.process_name][tree_node.timeperiod] = tree_node

        else:
            tree_node.job_record.state = job.STATE_EMBRYO
            msg = 'Transferred job record %s for %s in timeperiod %s to %s;' \
                  % (tree_node.job_record.db_id,
#.........这里部分代码省略.........

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class AbstractUowAwareWorker(AbstractMqWorker):
    """ Abstract class is inherited by all workers/aggregators
    that are aware of unit_of_work and capable of processing it """

    def __init__(self, process_name, perform_db_logging=False):
        super(AbstractUowAwareWorker, self).__init__(process_name)
        self.perform_db_logging = perform_db_logging
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.mq_transmitter = MqTransmitter(self.logger)

    def __del__(self):
        del self.mq_transmitter
        super(AbstractUowAwareWorker, self).__del__()

    # **************** Abstract Methods ************************
    def _init_performance_tracker(self, logger):
        self.performance_tracker = UowAwareTracker(logger)
        self.performance_tracker.start()

    def _process_uow(self, uow):
        """
        :param uow: unit_of_work to process
        :return: a tuple (number of processed items/documents/etc, desired unit_of_work state) or None
        if None is returned then it is assumed that the return tuple is (0, unit_of_work.STATE_PROCESSED)
        :raise an Exception if the UOW shall be marked as STATE_INVALID
        """
        raise NotImplementedError('method _process_uow must be implemented by {0}'.format(self.__class__.__name__))

    def _clean_up(self):
        """ method is called from the *finally* clause and is suppose to clean up after the uow processing """
        pass

    def _mq_callback(self, message):
        try:
            mq_request = MqTransmission.from_json(message.body)
            uow = self.uow_dao.get_one(mq_request.record_db_id)
            if not uow.is_requested:
                # accept only UOW in STATE_REQUESTED
                self.logger.warning('Skipping UOW: id {0}; state {1};'.format(message.body, uow.state),
                                    exc_info=False)
                self.consumer.acknowledge(message.delivery_tag)
                return
        except Exception:
            self.logger.error('Safety fuse. Can not identify UOW {0}'.format(message.body), exc_info=True)
            self.consumer.acknowledge(message.delivery_tag)
            return

        db_log_handler = UowLogHandler(self.logger, uow.db_id)
        try:
            uow.state = unit_of_work.STATE_IN_PROGRESS
            uow.started_at = datetime.utcnow()
            self.uow_dao.update(uow)
            self.performance_tracker.start_uow(uow)

            if self.perform_db_logging:
                db_log_handler.attach()

            result = self._process_uow(uow)
            if result is None:
                self.logger.warning('method {0}._process_uow returned None. Assuming happy flow.'
                                    .format(self.__class__.__name__))
                number_of_aggregated_objects, target_state = 0, unit_of_work.STATE_PROCESSED
            else:
                number_of_aggregated_objects, target_state = result

            uow.number_of_aggregated_documents = number_of_aggregated_objects
            uow.number_of_processed_documents = self.performance_tracker.success_per_job
            uow.finished_at = datetime.utcnow()
            uow.state = target_state
            self.uow_dao.update(uow)

            if uow.is_finished:
                self.performance_tracker.finish_uow()
            else:
                self.performance_tracker.cancel_uow()

        except Exception as e:
            fresh_uow = self.uow_dao.get_one(mq_request.record_db_id)
            self.performance_tracker.cancel_uow()
            if fresh_uow.is_canceled:
                self.logger.warning('UOW {0} for {1}@{2} was likely marked by MX as SKIPPED. No UOW update performed.'
                                    .format(uow.db_id, uow.process_name, uow.timeperiod), exc_info=False)
            else:
                self.logger.error('Safety fuse while processing UOW {0} for {1}@{2}: {3}'
                                  .format(uow.db_id, uow.process_name, uow.timeperiod, e), exc_info=True)
                uow.state = unit_of_work.STATE_INVALID
                self.uow_dao.update(uow)

        finally:
            self.consumer.acknowledge(message.delivery_tag)
            self.consumer.close()
            self._clean_up()
            db_log_handler.detach()

        try:
            self.mq_transmitter.publish_uow_status(uow)
            self.logger.info('UOW *{0}* status report published into MQ'.format(uow.state))
        except Exception:
            self.logger.error('Error on UOW status report publishing', exc_info=True)

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class StatusBusListener(object):
    """ class instance listens to the QUEUE_UOW_REPORT queue and updates Timetable records correspondingly """

    def __init__(self, scheduler):
        self.scheduler = scheduler
        self.timetable = scheduler.timetable
        self.logger = scheduler.logger
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.consumer = Consumer(QUEUE_UOW_REPORT)
        self.main_thread = None

    def __del__(self):
        try:
            self.logger.info('StatusBusListener: Closing Flopsy Consumer...')
            self.consumer.close()
        except Exception as e:
            self.logger.error('StatusBusListener: Exception caught while closing Flopsy Consumer: %s' % str(e))

    # ********************** thread-related methods ****************************
    def _mq_callback(self, message):
        """ method processes messages from Synergy Worker and updates corresponding Timetable record,
        as well as the job itself
        :param message: <SynergyMqTransmission> mq message """
        try:
            self.logger.info('StatusBusListener {')

            mq_request = SynergyMqTransmission.from_json(message.body)
            uow = self.uow_dao.get_one(mq_request.unit_of_work_id)
            if uow.unit_of_work_type != TYPE_MANAGED:
                self.logger.info('Received transmission from TYPE_FREERUN execution. Ignoring it.')
                return

            tree = self.timetable.get_tree(uow.process_name)
            node = tree.get_node(uow.process_name, uow.timeperiod)

            if uow.db_id != node.job_record.related_unit_of_work:
                self.logger.info('Received transmission is likely outdated. Ignoring it.')
                return

            if not uow.is_finished:
                # rely on Garbage Collector to re-trigger the failing unit_of_work
                self.logger.info('Received unit_of_work status report from %s at %s in non-final state %s. Ignoring it.'
                                 % (uow.process_name, uow.timeperiod, uow.state))
                return

            process_entry = self.scheduler.managed_handlers[uow.process_name].process_entry
            state_machine = self.scheduler.state_machines[process_entry.state_machine_name]
            assert isinstance(state_machine, AbstractStateMachine)

            self.logger.info('Commencing shallow state update for unit_of_work from %s at %s in %s.'
                             % (uow.process_name, uow.timeperiod, uow.state))
            state_machine.shallow_state_update(uow)

        except KeyError:
            self.logger.error('Access error for %s' % str(message.body), exc_info=True)
        except LookupError:
            self.logger.error('Can not perform shallow state update for %s' % str(message.body), exc_info=True)
        except Exception:
            self.logger.error('Unexpected error during shallow state update for %s' % str(message.body), exc_info=True)
        finally:
            self.consumer.acknowledge(message.delivery_tag)
            self.logger.info('StatusBusListener }')

    def _run_mq_listener(self):
        try:
            self.consumer.register(self._mq_callback)
            self.logger.info('StatusBusListener: instantiated and activated.')
            self.consumer.wait()
        except (AMQPError, IOError) as e:
            self.logger.error('StatusBusListener: AMQPError %s' % str(e))
        finally:
            self.__del__()
            self.logger.info('StatusBusListener: Shutting down... All auxiliary threads stopped.')

    def start(self, *_):
        self.main_thread = Thread(target=self._run_mq_listener)
        self.main_thread.daemon = True
        self.main_thread.start()

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]

#.........这里部分代码省略.........
                 False, if time_grouping is custom but the given timeperiod matches the grouped timeperiod.
                 True, if the timeperiod falls in-between grouping cracks
        """
        time_grouping = context.process_context[process_name].time_grouping
        if time_grouping == 1:
            return False

        process_hierarchy = self.timetable.get_tree(process_name).process_hierarchy
        timeperiod_dict = process_hierarchy[process_name].timeperiod_dict
        return timeperiod_dict._translate_timeperiod(timeperiod) != timeperiod

    def _process_noop_timeperiod(self, job_record):
        """ method is valid for processes having time_grouping != 1.
            should a job record fall in-between grouped time milestones,
            its state should be set to STATE_NOOP without any processing """
        job_record.state = job.STATE_NOOP
        self.job_dao.update(job_record)

        time_grouping = context.process_context[job_record.process_name].time_grouping
        msg = 'Job {0}@{1} with time_grouping {2} was transferred to STATE_NOOP' \
              .format(job_record.process_name, job_record.timeperiod, time_grouping)
        self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)

    def _process_state_embryo(self, job_record):
        """ method that takes care of processing job records in STATE_EMBRYO state"""
        pass

    def _process_state_in_progress(self, job_record):
        """ method that takes care of processing job records in STATE_IN_PROGRESS state"""
        pass

    def _process_state_final_run(self, job_record):
        """method takes care of processing job records in STATE_FINAL_RUN state"""
        uow = self.uow_dao.get_one(job_record.related_unit_of_work)
        if uow.is_processed:
            self.update_job(job_record, uow, job.STATE_PROCESSED)
        elif uow.is_noop:
            self.update_job(job_record, uow, job.STATE_NOOP)
        elif uow.is_canceled:
            self.update_job(job_record, uow, job.STATE_SKIPPED)
        elif uow.is_invalid:
            msg = 'Job {0}: UOW for {1}@{2} is in {3}; ' \
                  'relying on the Garbage Collector to either recycle or cancel the UOW.' \
                  .format(job_record.db_id, job_record.process_name, job_record.timeperiod, uow.state)
            self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)
        else:
            msg = 'Suppressed creating UOW for {0}@{1}; Job is in {2}; uow is in {3}' \
                  .format(job_record.process_name, job_record.timeperiod, job_record.state, uow.state)
            self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)

        timetable_tree = self.timetable.get_tree(job_record.process_name)
        timetable_tree.build_tree()

    def _process_terminal_state(self, job_record):
        """ method logs a warning message notifying that the job is no longer govern by this state machine """
        msg = 'Job {0} for {1}@{2} is in the terminal state {3}, ' \
              'and is no further govern by the State Machine {4}' \
              .format(job_record.db_id, job_record.process_name, job_record.timeperiod, job_record.state, self.name)
        self._log_message(WARNING, job_record.process_name, job_record.timeperiod, msg)

    def manage_job_with_blocking_children(self, job_record):
        """ method will trigger job processing only if:
            - all children are finished (STATE_PROCESSED, STATE_SKIPPED, STATE_NOOP)
            - all external dependencies are finalized (STATE_PROCESSED, STATE_SKIPPED, STATE_NOOP) """
        is_job_finalizable = self.timetable.is_job_record_finalizable(job_record)
        if is_job_finalizable:

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class AbstractActionHandler(BaseRequestHandler):
    def __init__(self, request, **values):
        super(AbstractActionHandler, self).__init__(request, **values)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.log_recording_dao = LogRecordingDao(self.logger)

    @property
    def thread_handler(self):
        raise NotImplementedError('property thread_handler must be implemented by {0}'.format(self.__class__.__name__))

    @property
    def process_entry(self):
        raise NotImplementedError('property process_entry must be implemented by {0}'.format(self.__class__.__name__))

    @property
    def uow_id(self):
        raise NotImplementedError('property uow_id must be implemented by {0}'.format(self.__class__.__name__))

    def get_event_log(self):
        raise NotImplementedError('method action_get_event_log must be implemented by {0}'
                                  .format(self.__class__.__name__))

    @safe_json_response
    def get_uow(self):
        if self.uow_id is None:
            resp = {'response': 'no related unit_of_work'}
        else:
            resp = self.uow_dao.get_one(self.uow_id).document
        return resp

    @safe_json_response
    def get_uow_log(self):
        try:
            resp = self.log_recording_dao.get_one(self.uow_id).document
        except (TypeError, LookupError):
            resp = {'response': 'no related uow log'}
        return resp

    @valid_action_request
    def change_interval(self):
        resp = dict()
        new_interval = self.request_arguments['interval']
        if new_interval is not None:
            self.thread_handler.change_interval(new_interval)
            msg = 'changed interval for {0} to {1}'.format(self.thread_handler.key, new_interval)
            self.logger.info('MX: {0}'.format(msg))
            resp['status'] = msg

        return resp

    @valid_action_request
    def trigger_now(self):
        self.thread_handler.trigger()
        self.logger.info('MX: triggered thread handler {0}'.format(self.thread_handler.key))
        return self.reply_ok()

    @valid_action_request
    def activate_trigger(self):
        self.thread_handler.activate()
        self.logger.info('MX: activated thread handler {0}'.format(self.thread_handler.key))
        return self.reply_ok()

    @valid_action_request
    def deactivate_trigger(self):
        self.thread_handler.deactivate()
        self.logger.info('MX: deactivated thread handler {0}'.format(self.thread_handler.key))
        return self.reply_ok()

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class StateMachineFreerun(object):
    """ State Machine to handle freerun jobs/triggers """

    def __init__(self, logger, name=STATE_MACHINE_FREERUN):
        self.name = name
        self.logger = logger
        self.mq_transmitter = MqTransmitter(self.logger)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.sfe_dao = FreerunProcessDao(self.logger)

    @with_reconnect
    def _log_message(self, level, freerun_entry, msg):
        """ method performs logging into log file and the freerun_entry """
        self.logger.log(level, msg)

        assert isinstance(freerun_entry, FreerunProcessEntry)
        event_log = freerun_entry.event_log
        if len(event_log) > MAX_NUMBER_OF_EVENTS:
            del event_log[-1]
        event_log.insert(0, msg)
        self.sfe_dao.update(freerun_entry)

    @with_reconnect
    def _insert_uow(self, freerun_entry):
        """ creates unit_of_work and inserts it into the DB
            :raise DuplicateKeyError: if unit_of_work with given parameters already exists """
        current_timeperiod = time_helper.actual_timeperiod(QUALIFIER_REAL_TIME)
        process_entry = context.process_context[freerun_entry.process_name]

        uow = UnitOfWork()
        uow.process_name = freerun_entry.schedulable_name
        uow.timeperiod = current_timeperiod
        uow.start_id = 0
        uow.end_id = 0
        uow.start_timeperiod = current_timeperiod
        uow.end_timeperiod = current_timeperiod
        uow.created_at = datetime.utcnow()
        uow.submitted_at = datetime.utcnow()
        uow.source = process_entry.source if hasattr(process_entry, 'source') else None
        uow.sink = process_entry.sink if hasattr(process_entry, 'sink') else None
        uow.state = unit_of_work.STATE_REQUESTED
        uow.unit_of_work_type = unit_of_work.TYPE_FREERUN
        uow.number_of_retries = 0
        uow.arguments = freerun_entry.arguments
        uow.db_id = self.uow_dao.insert(uow)

        msg = 'Created: UOW {0} for {1}@{2}.' \
              .format(uow.db_id, freerun_entry.schedulable_name, current_timeperiod)
        self._log_message(INFO, freerun_entry, msg)
        return uow

    def _publish_uow(self, freerun_entry, uow):
        self.mq_transmitter.publish_freerun_uow(freerun_entry, uow)
        msg = 'Published: UOW {0} for {1}.'.format(uow.db_id, freerun_entry.schedulable_name)
        self._log_message(INFO, freerun_entry, msg)

    def insert_and_publish_uow(self, freerun_entry):
        try:
            uow = self._insert_uow(freerun_entry)
        except DuplicateKeyError as e:
            msg = 'Duplication of UOW found for {0}. Error msg: {1}'.format(freerun_entry.schedulable_name, e)
            self._log_message(WARNING, freerun_entry, msg)
            uow = self.uow_dao.recover_from_duplicatekeyerror(e)

        if uow is not None:
            # publish the created/caught up unit_of_work
            self._publish_uow(freerun_entry, uow)
            freerun_entry.related_unit_of_work = uow.db_id
            self.sfe_dao.update(freerun_entry)
        else:
            msg = 'PERSISTENT TIER ERROR! Unable to locate UOW for {0}' \
                  .format(freerun_entry.schedulable_name)
            self._log_message(WARNING, freerun_entry, msg)

    def _process_state_embryo(self, freerun_entry):
        """ method creates unit_of_work and associates it with the FreerunProcessEntry """
        self.insert_and_publish_uow(freerun_entry)

    def _process_state_in_progress(self, freerun_entry, uow):
        """ method that takes care of processing unit_of_work records in STATE_REQUESTED or STATE_IN_PROGRESS states"""
        self._publish_uow(freerun_entry, uow)

    def _process_terminal_state(self, freerun_entry, uow):
        """ method that takes care of processing unit_of_work records in
            STATE_PROCESSED, STATE_NOOP, STATE_INVALID, STATE_CANCELED states"""
        msg = 'UOW for {0} found in state {1}.'.format(freerun_entry.schedulable_name, uow.state)
        self._log_message(INFO, freerun_entry, msg)
        self.insert_and_publish_uow(freerun_entry)

    def manage_schedulable(self, freerun_entry):
        """ method main duty - is to _avoid_ publishing another unit_of_work, if previous was not yet processed
        In case the Scheduler sees that the unit_of_work is pending it will fire another WorkerMqRequest """

        assert isinstance(freerun_entry, FreerunProcessEntry)
        if freerun_entry.related_unit_of_work is None:
            uow = None
        else:
            uow = self.uow_dao.get_one(freerun_entry.related_unit_of_work)

        try:
#.........这里部分代码省略.........

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class GarbageCollector(object):
    """ GC is triggered directly by Synergy Scheduler.
        It scans for invalid or stalled unit_of_work and re-triggers them.
        GC is vital for the health of the system.
        Deployment with no GC is considered invalid """

    def __init__(self, scheduler):
        self.logger = get_logger(PROCESS_GC, append_to_console=False, redirect_stdstream=False)
        self.managed_handlers = scheduler.managed_handlers
        self.mq_transmitter = MqTransmitter(self.logger)
        self.timetable = scheduler.timetable

        self.lock = Lock()
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.reprocess_uows = collections.defaultdict(PriorityQueue)
        self.timer = RepeatTimer(settings.settings['gc_run_interval'], self._run)

    @thread_safe
    def scan_uow_candidates(self):
        """ method performs two actions:
            - enlist stale or invalid units of work into reprocessing queue
            - cancel UOWs that are older than 2 days and have been submitted more than 1 hour ago """
        try:
            since = settings.settings['synergy_start_timeperiod']
            uow_list = self.uow_dao.get_reprocessing_candidates(since)
        except LookupError as e:
            self.logger.info('flow: no UOW candidates found for reprocessing: {0}'.format(e))
            return

        for uow in uow_list:
            try:
                if uow.process_name not in self.managed_handlers:
                    self.logger.debug('process {0} is not known to the Synergy Scheduler. Skipping its UOW.'
                                      .format(uow.process_name))
                    continue

                thread_handler = self.managed_handlers[uow.process_name]
                assert isinstance(thread_handler, ManagedThreadHandler)

                if not thread_handler.process_entry.is_on:
                    self.logger.debug('process {0} is inactive. Skipping its UOW.'.format(uow.process_name))
                    continue

                entry = PriorityEntry(uow)
                if entry in self.reprocess_uows[uow.process_name]:
                    # given UOW is already registered in the reprocessing queue
                    continue

                # ASSUMPTION: UOW is re-created by a state machine during reprocessing
                # thus - any UOW older 2 days could be marked as STATE_CANCELED
                if datetime.utcnow() - uow.created_at > timedelta(hours=settings.settings['gc_life_support_hours']):
                    self._cancel_uow(uow)
                    continue

                # if the UOW has been idle for more than 1 hour - resubmit it
                if datetime.utcnow() - uow.submitted_at > timedelta(hours=settings.settings['gc_resubmit_after_hours'])\
                        or uow.is_invalid:
                    # enlist the UOW into the reprocessing queue
                    self.reprocess_uows[uow.process_name].put(entry)

            except Exception as e:
                self.logger.error('flow exception: {0}'.format(e), exc_info=True)

    def _flush_queue(self, q, ignore_priority=False):
        """
        :param q: PriorityQueue instance holding GarbageCollector entries
        :param ignore_priority: If True - all GarbageCollector entries should be resubmitted
                If False - only those entries whose waiting time has expired will be resubmitted
        """
        assert isinstance(q, PriorityQueue)

        current_timestamp = compute_release_time(lag_in_minutes=0)
        for _ in range(len(q)):
            entry = q.pop()
            assert isinstance(entry, PriorityEntry)

            if ignore_priority or entry.release_time < current_timestamp:
                self._resubmit_uow(entry.entry)
            else:
                q.put(entry)
                break

    @thread_safe
    def flush(self, ignore_priority=False):
        """ method iterates over each reprocessing queues and re-submits UOW whose waiting time has expired """
        for process_name, q in self.reprocess_uows.items():
            self._flush_queue(q, ignore_priority)

    @thread_safe
    def validate(self):
        """ method iterates over the reprocessing queue and synchronizes state of every UOW with the DB
            should it change via the MX to STATE_CANCELED - remove the UOW from the queue """
        for process_name, q in self.reprocess_uows.items():
            if not q:
                continue

            invalid_entries = list()
            for entry in q.queue:
                assert isinstance(entry, PriorityEntry)
                uow = self.uow_dao.get_one(entry.entry.db_id)
#.........这里部分代码省略.........

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class AbstractUowAwareWorker(AbstractMqWorker):
    """ Abstract class is inherited by all workers/aggregators
    that are aware of unit_of_work and capable of processing it"""

    def __init__(self, process_name):
        super(AbstractUowAwareWorker, self).__init__(process_name)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.publishers = PublishersPool(self.logger)

    def __del__(self):
        self.publishers.close()
        super(AbstractUowAwareWorker, self).__del__()

    # **************** Abstract Methods ************************
    def _init_performance_ticker(self, logger):
        self.performance_ticker = UowAwareTracker(logger)
        self.performance_ticker.start()

    def _process_uow(self, uow):
        """
        :param uow: unit_of_work to process
        :return: a tuple (number of processed items/documents/etc, desired unit_of_work state) or None
        if None is returned then it is assumed that the return tuple is (0, unit_of_work.STATE_PROCESSED)
        :raise an Exception if the UOW shall be marked as STATE_INVALID
        """
        raise NotImplementedError('_process_uow must be overridden in the child class %s' % self.__class__.__name__)

    def _clean_up(self):
        """ method is called from the *finally* clause and is suppose to clean up after the uow processing """
        pass

    def _mq_callback(self, message):
        try:
            mq_request = SynergyMqTransmission.from_json(message.body)
            uow = self.uow_dao.get_one(mq_request.unit_of_work_id)
            if not uow.is_requested:
                # accept only UOW in STATE_REQUESTED
                self.logger.warn('Skipping unit_of_work: id %s; state %s;' % (str(message.body), uow.state),
                                 exc_info=False)
                self.consumer.acknowledge(message.delivery_tag)
                return
        except Exception:
            self.logger.error('Safety fuse. Can not identify unit_of_work %s' % str(message.body), exc_info=True)
            self.consumer.acknowledge(message.delivery_tag)
            return

        try:
            uow.state = unit_of_work.STATE_IN_PROGRESS
            uow.started_at = datetime.utcnow()
            self.uow_dao.update(uow)
            self.performance_ticker.start_uow(uow)

            result = self._process_uow(uow)
            if result is None or isinstance(result, str):
                number_of_aggregated_objects, target_state = 0, unit_of_work.STATE_PROCESSED
            else:
                number_of_aggregated_objects, target_state = result

            uow.number_of_aggregated_documents = number_of_aggregated_objects
            uow.number_of_processed_documents = self.performance_ticker.per_job
            uow.finished_at = datetime.utcnow()
            uow.state = target_state
            self.uow_dao.update(uow)
            self.performance_ticker.finish_uow()
        except Exception as e:
            fresh_uow = self.uow_dao.get_one(mq_request.unit_of_work_id)
            self.performance_ticker.cancel_uow()
            if fresh_uow.is_canceled:
                self.logger.warn('unit_of_work: id %s was likely marked by MX as SKIPPED. '
                                 'No unit_of_work update is performed.' % str(message.body),
                                 exc_info=False)
            else:
                self.logger.error('Safety fuse while processing unit_of_work %s in timeperiod %s : %r'
                                  % (message.body, uow.timeperiod, e), exc_info=True)
                uow.state = unit_of_work.STATE_INVALID
                self.uow_dao.update(uow)

        finally:
            self.consumer.acknowledge(message.delivery_tag)
            self.consumer.close()
            self._clean_up()

        try:
            publisher = self.publishers.get(QUEUE_UOW_REPORT)
            publisher.publish(mq_request.document)
            publisher.release()
            self.logger.info('Published unit_of_work status report into %s queue' % QUEUE_UOW_REPORT)
        except Exception:
            self.logger.error('Error on unit_of_work status report publishing', exc_info=True)

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class UowStatusListener(object):
    """ class instance listens to the QUEUE_UOW_STATUS queue and updates Timetable records correspondingly """

    def __init__(self, scheduler):
        self.scheduler = scheduler
        self.timetable = scheduler.timetable
        self.logger = scheduler.logger
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.consumer = Consumer(QUEUE_UOW_STATUS)
        self.main_thread = None

    def __del__(self):
        try:
            self.logger.info('UowStatusListener: Closing Flopsy Consumer...')
            self.consumer.close()
        except Exception as e:
            self.logger.error('UowStatusListener: Exception caught while closing Flopsy Consumer: {0}'.format(e))

    # ********************** thread-related methods ****************************
    def _mq_callback(self, message):
        """ method processes messages from Synergy Worker and updates corresponding Timetable record,
            as well as the job itself
            :param message: <MqTransmission> mq message """
        try:
            self.logger.info('UowStatusListener {')

            mq_request = MqTransmission.from_json(message.body)
            uow = self.uow_dao.get_one(mq_request.record_db_id)
            if uow.unit_of_work_type != unit_of_work.TYPE_MANAGED:
                self.logger.info('Received transmission from non-managed UOW execution: {0}. Ignoring it.'
                                 .format(uow.unit_of_work_type))
                return

            tree = self.timetable.get_tree(uow.process_name)
            node = tree.get_node(uow.process_name, uow.timeperiod)

            if uow.db_id != node.job_record.related_unit_of_work:
                self.logger.info('Received transmission is likely outdated. Ignoring it.')
                return

            if not uow.is_finished:
                # rely on Garbage Collector to re-trigger the failing unit_of_work
                self.logger.info('Received transmission from {0}@{1} in non-final state {2}. Ignoring it.'
                                 .format(uow.process_name, uow.timeperiod, uow.state))
                return

            state_machine = self.scheduler.state_machine_for(node.process_name)
            self.logger.info('Commencing StateMachine.notify with UOW from {0}@{1} in {2}.'
                             .format(uow.process_name, uow.timeperiod, uow.state))
            state_machine.notify(uow)

        except KeyError:
            self.logger.error('Access error for {0}'.format(message.body), exc_info=True)
        except Exception:
            self.logger.error('Error during StateMachine.notify call {0}'.format(message.body), exc_info=True)
        finally:
            self.consumer.acknowledge(message.delivery_tag)
            self.logger.info('UowStatusListener }')

    def _run_mq_listener(self):
        try:
            self.consumer.register(self._mq_callback)
            self.logger.info('UowStatusListener: instantiated and activated.')
            self.consumer.wait()
        except (AMQPError, IOError) as e:
            self.logger.error('UowStatusListener: AMQPError {0}'.format(e))
        finally:
            self.__del__()
            self.logger.info('UowStatusListener: Shut down.')

    def start(self, *_):
        self.main_thread = Thread(target=self._run_mq_listener)
        self.main_thread.daemon = True
        self.main_thread.start()

    def stop(self):
        """ method stops currently MQ Consumer listener, if any """
        self.__del__()

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class FreerunActionHandler(AbstractActionHandler):
    def __init__(self, request, **values):
        super(FreerunActionHandler, self).__init__(request, **values)
        self.process_name = self.request_arguments.get('process_name')
        self.entry_name = self.request_arguments.get('entry_name')
        self.freerun_process_dao = FreerunProcessDao(self.logger)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.is_request_valid = True if self.process_name and self.entry_name else False

        if self.is_request_valid:
            self.process_name = self.process_name.strip()
            self.entry_name = self.entry_name.strip()
            self.is_requested_state_on = 'is_on' in self.request_arguments and self.request_arguments['is_on']

    @AbstractActionHandler.thread_handler.getter
    def thread_handler(self):
        handler_key = (self.process_name, self.entry_name)
        return self.scheduler.freerun_handlers[handler_key]

    @AbstractActionHandler.process_entry.getter
    def process_entry(self):
        return self.thread_handler.process_entry

    @valid_action_request
    def action_cancel_uow(self):
        uow_id = self.process_entry.related_unit_of_work
        if uow_id is None:
            resp = {'response': 'no related unit_of_work'}
        else:
            uow = self.uow_dao.get_one(uow_id)
            uow.state = unit_of_work.STATE_CANCELED
            self.uow_dao.update(uow)
            resp = {'response': 'updated unit_of_work %r' % uow_id}
        return resp

    @valid_action_request
    def action_get_uow(self):
        uow_id = self.process_entry.related_unit_of_work
        if uow_id is None:
            resp = {'response': 'no related unit_of_work'}
        else:
            resp = self.uow_dao.get_one(uow_id).document
            for key in resp:
                resp[key] = str(resp[key])
        return resp

    @valid_action_request
    def action_get_log(self):
        return {'log': self.process_entry.log}

    @valid_action_request
    def action_update_entry(self):
        if 'insert_button' in self.request_arguments:
            process_entry = FreerunProcessEntry()
            process_entry.process_name = self.process_name
            process_entry.entry_name = self.entry_name

            if self.request_arguments['arguments']:
                arguments = self.request_arguments['arguments'].decode('unicode-escape')
                process_entry.arguments = json.loads(arguments)
            else:
                process_entry.arguments = {}

            process_entry.description = self.request_arguments['description']
            process_entry.is_on = self.is_requested_state_on
            process_entry.trigger_frequency = self.request_arguments['trigger_frequency']
            self.freerun_process_dao.update(process_entry)

            self.scheduler._register_process_entry(process_entry, self.scheduler.fire_freerun_worker)

        elif 'update_button' in self.request_arguments:
            is_interval_changed = self.process_entry.trigger_frequency != self.request_arguments['trigger_frequency']

            if self.request_arguments['arguments']:
                arguments = self.request_arguments['arguments'].decode('unicode-escape')
                self.process_entry.arguments = json.loads(arguments)
            else:
                self.process_entry.arguments = {}

            self.process_entry.description = self.request_arguments['description']
            self.process_entry.is_on = self.is_requested_state_on
            self.process_entry.trigger_frequency = self.request_arguments['trigger_frequency']
            self.freerun_process_dao.update(self.process_entry)

            if is_interval_changed:
                self.action_change_interval()

            if self.process_entry.is_on != self.is_requested_state_on:
                if self.is_requested_state_on:
                    self.action_activate_trigger()
                else:
                    self.action_deactivate_trigger()

        elif 'delete_button' in self.request_arguments:
            handler_key = (self.process_name, self.entry_name)
            self.thread_handler.deactivate()
            self.freerun_process_dao.remove(handler_key)
            del self.scheduler.freerun_handlers[handler_key]

        elif 'cancel_button' in self.request_arguments:
#.........这里部分代码省略.........

# 需要导入模块: from synergy.db.dao.unit_of_work_dao import UnitOfWorkDao [as 别名]
# 或者: from synergy.db.dao.unit_of_work_dao.UnitOfWorkDao import get_one [as 别名]
class StateMachineFreerun(object):
    """ State Machine to handle freerun jobs/triggers """

    def __init__(self, logger, name=STATE_MACHINE_FREERUN):
        self.name = name
        self.logger = logger
        self.publishers = PublishersPool(self.logger)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.sfe_dao = FreerunProcessDao(self.logger)

    def __del__(self):
        try:
            self.logger.info('Closing Flopsy Publishers Pool...')
            self.publishers.close()
        except Exception as e:
            self.logger.error('Exception caught while closing Flopsy Publishers Pool: %s' % str(e))

    @with_reconnect
    def _log_message(self, level, freerun_entry, msg):
        """ method performs logging into log file and the freerun_entry """
        self.logger.log(level, msg)

        assert isinstance(freerun_entry, FreerunProcessEntry)
        log = freerun_entry.log
        if len(log) > MAX_NUMBER_OF_LOG_ENTRIES:
            del log[-1]
        log.insert(0, msg)
        self.sfe_dao.update(freerun_entry)

    @with_reconnect
    def _insert_uow(self, freerun_entry):
        """ creates unit_of_work and inserts it into the DB
            :raise DuplicateKeyError: if unit_of_work with given parameters already exists """
        current_timeperiod = time_helper.actual_timeperiod(QUALIFIER_REAL_TIME)

        uow = UnitOfWork()
        uow.process_name = freerun_entry.schedulable_name
        uow.timeperiod = current_timeperiod
        uow.start_id = 0
        uow.end_id = 0
        uow.start_timeperiod = current_timeperiod
        uow.end_timeperiod = current_timeperiod
        uow.created_at = datetime.utcnow()
        uow.source = context.process_context[freerun_entry.process_name].source
        uow.sink = context.process_context[freerun_entry.process_name].sink
        uow.state = unit_of_work.STATE_REQUESTED
        uow.unit_of_work_type = TYPE_FREERUN
        uow.number_of_retries = 0
        uow.arguments = freerun_entry.arguments
        uow.db_id = self.uow_dao.insert(uow)

        msg = 'Created: UOW %s for %s in timeperiod %s.' \
              % (uow.db_id, freerun_entry.schedulable_name, current_timeperiod)
        self._log_message(INFO, freerun_entry, msg)
        return uow

    def _publish_uow(self, freerun_entry, uow):
        mq_request = SynergyMqTransmission(process_name=freerun_entry.process_name,
                                           entry_name=freerun_entry.entry_name,
                                           unit_of_work_id=uow.db_id)

        publisher = self.publishers.get(freerun_entry.process_name)
        publisher.publish(mq_request.document)
        publisher.release()

        msg = 'Published: UOW %s for %s.' % (uow.db_id, freerun_entry.schedulable_name)
        self._log_message(INFO, freerun_entry, msg)

    def insert_and_publish_uow(self, freerun_entry):
        try:
            uow = self._insert_uow(freerun_entry)
        except DuplicateKeyError as e:
            msg = 'Duplication of unit_of_work found for %s. Error msg: %r' % (freerun_entry.schedulable_name, e)
            self._log_message(WARNING, freerun_entry, msg)
            uow = self.uow_dao.recover_from_duplicatekeyerror(e)

        if uow is not None:
            # publish the created/caught up unit_of_work
            self._publish_uow(freerun_entry, uow)
            freerun_entry.related_unit_of_work = uow.db_id
            self.sfe_dao.update(freerun_entry)
        else:
            msg = 'SYSTEM IS LIKELY IN UNSTABLE STATE! Unable to locate unit_of_work for %s' \
                  % freerun_entry.schedulable_name
            self._log_message(WARNING, freerun_entry, msg)

    def manage_schedulable(self, freerun_entry):
        """ method main duty - is to _avoid_ publishing another unit_of_work, if previous was not yet processed
        In case the Scheduler sees that the unit_of_work is pending it will fire another WorkerMqRequest """

        assert isinstance(freerun_entry, FreerunProcessEntry)
        if freerun_entry.related_unit_of_work is None:
            uow = None
        else:
            uow = self.uow_dao.get_one(freerun_entry.related_unit_of_work)

        try:
            if uow is None:
                self._process_state_embryo(freerun_entry)

#.........这里部分代码省略.........