Python unit_of_work_dao.UnitOfWorkDao类代码示例

def create_and_insert_unit_of_work(process_name, start_id, end_id, state=unit_of_work.STATE_REQUESTED,
                                   timeperiod='INVALID_TIMEPERIOD'):
    """ method creates and inserts a unit_of_work into DB
    :return id of the created object in the db"""
    uow = create_unit_of_work(process_name, start_id, end_id, timeperiod, state)
    logger = get_logger(process_name)
    uow_dao = UnitOfWorkDao(logger)
    uow_id = uow_dao.insert(uow)
    return uow_id

 def __init__(self, process_name):
     super(GarbageCollectorWorker, self).__init__(process_name)
     self.lock = Lock()
     self.publishers = PublishersPool(self.logger)
     self.uow_dao = UnitOfWorkDao(self.logger)
     self.managed_dao = ManagedProcessDao(self.logger)
     self.managed_entries = dict()

 def __init__(self, scheduler):
     self.scheduler = scheduler
     self.timetable = scheduler.timetable
     self.logger = scheduler.logger
     self.uow_dao = UnitOfWorkDao(self.logger)
     self.consumer = Consumer(QUEUE_UOW_REPORT)
     self.main_thread = None

 def __init__(self, logger, timetable, name):
     self.name = name
     self.logger = logger
     self.publishers = PublishersPool(self.logger)
     self.timetable = timetable
     self.uow_dao = UnitOfWorkDao(self.logger)
     self.job_dao = JobDao(self.logger)

class FreerunStatements(object):
    def __init__(self, logger, freerun_handlers):
        self.lock = RLock()
        self.logger = logger
        self.freerun_handlers = freerun_handlers
        self.uow_dao = UnitOfWorkDao(self.logger)

    @thread_safe
    def retrieve_records(self, timeperiod, include_running,
                         include_processed, include_noop, include_failed, include_disabled):
        """ method looks for suitable UOW records and returns them as a dict"""
        resp = dict()
        try:
            query = unit_of_work_dao.QUERY_GET_FREERUN_SINCE(timeperiod, include_running,
                                                             include_processed, include_noop, include_failed)
            records_list = self.uow_dao.run_query(query)
            if len(records_list) == 0:
                self.logger.warning('MX: no Freerun UOW records found since {0}.'.format(timeperiod))

            for uow_record in records_list:
                if uow_record.process_name not in self.freerun_handlers:
                    continue

                thread_handler = self.freerun_handlers[uow_record.process_name]
                if not include_disabled and not thread_handler.process_entry.is_on:
                    continue

                resp[uow_record.key] = uow_record.document
        except Exception as e:
            self.logger.error('MX Dashboard FreerunStatements error: {0}'.format(e))
        return resp

 def setUp(self):
     self.process_name = PROCESS_ALERT_DAILY
     self.logger = get_logger(self.process_name)
     self.uow_id = create_and_insert_unit_of_work(self.process_name, 'range_start', 'range_end')
     self.uow_id = str(self.uow_id)
     self.uow_dao = UnitOfWorkDao(self.logger)
     self.uow_log_dao = UowLogDao(self.logger)

 def __init__(self, logger, timetable, name):
     self.name = name
     self.logger = logger
     self.mq_transmitter = MqTransmitter(self.logger)
     self.timetable = timetable
     self.uow_dao = UnitOfWorkDao(self.logger)
     self.job_dao = JobDao(self.logger)

class UowLogHandlerUnitTest(unittest.TestCase):
    """ Test flow:
        1. create a UOW in the database
        2. emulate mq message
        3. call _mq_callback method
        4. validate that all the messages are now found in the uow_log record
        5. remove UOW and uow_log record
    """

    def setUp(self):
        self.process_name = PROCESS_ALERT_DAILY
        self.logger = get_logger(self.process_name)
        self.uow_id = create_and_insert_unit_of_work(self.process_name, 'range_start', 'range_end')
        self.uow_id = str(self.uow_id)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.uow_log_dao = UowLogDao(self.logger)

    def tearDown(self):
        self.uow_dao.remove(self.uow_id)
        self.uow_log_dao.remove(self.uow_id)

    def test_logging(self):
        self.worker = ChattyWorker(self.process_name)
        message = TestMessage(process_name=self.process_name, uow_id=self.uow_id)
        self.worker._mq_callback(message)

        uow_log = self.uow_log_dao.get_one(self.uow_id)
        messages = INFO_LOG_MESSAGES + WARN_LOG_MESSAGES  # + STD_MESSAGES
        self.assertLessEqual(len(messages), len(uow_log.log))

        for index, message in enumerate(messages):
            self.assertIn(message, uow_log.log[index])

    def test_exception_logging(self):
        self.worker = ExceptionWorker(self.process_name)
        message = TestMessage(process_name=self.process_name, uow_id=self.uow_id)
        self.worker._mq_callback(message)

        uow_log = self.uow_log_dao.get_one(self.uow_id)
        messages = ['Exception: Artificially triggered exception to test Uow Exception Logging',
                    'method ExceptionWorker._process_uow returned None. Assuming happy flow.',
                    'at INVALID_TIMEPERIOD: Success/Failure 0/0']

        for index, message in enumerate(messages):
            self.assertIn(message, uow_log.log[index])

    def __init__(self, scheduler):
        self.logger = get_logger(PROCESS_GC, append_to_console=False, redirect_stdstream=False)
        self.managed_handlers = scheduler.managed_handlers
        self.mq_transmitter = MqTransmitter(self.logger)
        self.timetable = scheduler.timetable

        self.lock = Lock()
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.reprocess_uows = collections.defaultdict(PriorityQueue)
        self.timer = RepeatTimer(settings.settings['gc_run_interval'], self._run)

    def __init__(self, logger, message, consumer, performance_ticker):
        self.logger = logger
        self.message = message
        self.mq_request = SynergyMqTransmission.from_json(message.body)
        self.consumer = consumer
        self.performance_ticker = performance_ticker
        self.alive = False
        self.return_code = -1
        self.uow_dao = UnitOfWorkDao(self.logger)

        self.thread_name = str(self.mq_request)
        super(BashRunnable, self).__init__(name=self.thread_name)

    def __init__(self, request, **values):
        super(FreerunActionHandler, self).__init__(request, **values)
        self.process_name = self.request_arguments.get('process_name')
        self.entry_name = self.request_arguments.get('entry_name')
        self.freerun_process_dao = FreerunProcessDao(self.logger)
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.is_request_valid = True if self.process_name and self.entry_name else False

        if self.is_request_valid:
            self.process_name = self.process_name.strip()
            self.entry_name = self.entry_name.strip()
            self.is_requested_state_on = 'is_on' in self.request_arguments and self.request_arguments['is_on']

    def __init__(self, logger):
        self.lock = RLock()
        self.logger = logger
        self.uow_dao = UnitOfWorkDao(self.logger)
        self.job_dao = JobDao(self.logger)
        self.reprocess = dict()

        # self.trees contain all of the trees and manages much of their life cycle
        # remember to enlist there all trees the system is working with
        self.trees = self._construct_trees_from_context()

        self._register_callbacks()
        self._register_dependencies()
        self.load_tree()
        self.build_trees()
        self.validate()

    def test_select_reprocessing_candidates(self):
        logger = get_logger(PROCESS_UNIT_TEST)
        uow_dao = UnitOfWorkDao(logger)

        try:
            initial_candidates = uow_dao.get_reprocessing_candidates()
        except:
            initial_candidates = []

        try:
            initial_positive_candidates = uow_dao.get_reprocessing_candidates('2010123123')
        except:
            initial_positive_candidates = []

        positive_timeperiods = {u'2010123123': PROCESS_SITE_HOURLY,    # hourly time qualifier
                                u'2010123100': PROCESS_SITE_DAILY,     # daily time qualifier
                                u'2010120000': PROCESS_SITE_MONTHLY,   # monthly time qualifier
                                u'2010000000': PROCESS_SITE_YEARLY}    # yearly time qualifier

        negative_timeperiods = {u'2009123123': PROCESS_SITE_HOURLY,    # hourly time qualifier
                                u'2009123100': PROCESS_SITE_DAILY,     # daily time qualifier
                                u'2009120000': PROCESS_SITE_MONTHLY,   # monthly time qualifier
                                u'2009000000': PROCESS_SITE_YEARLY}    # yearly time qualifier

        all_timeperiods = dict()
        all_timeperiods.update(positive_timeperiods)
        all_timeperiods.update(negative_timeperiods)

        created_uow = []
        for timeperiod, process_name in all_timeperiods.items():
            created_uow.append(create_and_insert_unit_of_work(process_name,
                                                              0,
                                                              1,
                                                              timeperiod=timeperiod,
                                                              state=unit_of_work.STATE_INVALID))

        candidates = uow_dao.get_reprocessing_candidates('2010123123')
        self.assertEqual(len(candidates) - len(initial_positive_candidates), len(positive_timeperiods))

        candidates = uow_dao.get_reprocessing_candidates()
        self.assertEqual(len(candidates) - len(initial_candidates), len(all_timeperiods))

        for uow_id in created_uow:
            uow_dao.remove(uow_id)

class FreerunStatements(object):
    def __init__(self, logger):
        self.lock = RLock()
        self.logger = logger
        self.uow_dao = UnitOfWorkDao(self.logger)

    @thread_safe
    def retrieve_records(self, timeperiod, unprocessed_only):
        """ method looks for suitable UOW records and returns them as a dict"""
        resp = dict()
        try:
            query = unit_of_work_dao.QUERY_GET_FREERUN_SINCE(timeperiod, unprocessed_only)
            records_list = self.uow_dao.run_query(query)
            if len(records_list) == 0:
                self.logger.warn('No Freerun UOW records found since {0}.'.format(timeperiod))

            for uow_record in records_list:
                resp[uow_record.key] = uow_record.document
        except Exception as e:
            self.logger.error('DashboardHandler error: {0}'.format(e))
        return resp

class BashRunnable(threading.Thread):
    """Process starts remote or local bash script job, supervises its execution and updates mq"""

    def __init__(self, logger, message, consumer, performance_ticker):
        self.logger = logger
        self.message = message
        self.mq_request = SynergyMqTransmission.from_json(message.body)
        self.consumer = consumer
        self.performance_ticker = performance_ticker
        self.alive = False
        self.return_code = -1
        self.uow_dao = UnitOfWorkDao(self.logger)

        self.thread_name = str(self.mq_request)
        super(BashRunnable, self).__init__(name=self.thread_name)

    def _poll_process(self):
        return self.alive, self.return_code

    def _start_process(self):
        try:
            uow = self.uow_dao.get_one(self.mq_request.unit_of_work_id)
            if not uow.is_requested:
                # accept only UOW in STATE_REQUESTED
                self.logger.warn('Skipping unit_of_work: id %s; state %s;' % (str(self.message.body), uow.state),
                                 exc_info=False)
                self.consumer.acknowledge(self.message.delivery_tag)
                return
        except Exception:
            self.logger.error('Safety fuse. Can not identify unit_of_work %s' % str(self.message.body), exc_info=True)
            self.consumer.acknowledge(self.message.delivery_tag)
            return

        try:
            self.logger.info('start: %s {' % self.thread_name)
            self.alive = True

            uow.state = unit_of_work.STATE_IN_PROGRESS
            uow.started_at = datetime.utcnow()
            self.uow_dao.update(uow)

            fabric.operations.env.warn_only = True
            fabric.operations.env.abort_on_prompts = True
            fabric.operations.env.use_ssh_config = True
            fabric.operations.env.host_string = uow.arguments[ARGUMENT_CMD_HOST]

            command = os.path.join(uow.arguments[ARGUMENT_CMD_PATH],
                                   uow.arguments[ARGUMENT_CMD_FILE])
            command += ' %s' % uow.arguments[ARGUMENT_CMD_ARGS]

            run_result = fabric.operations.run(command, pty=False)
            if run_result.succeeded:
                self.return_code = 0

            uow.finished_at = datetime.utcnow()
            uow.state = unit_of_work.STATE_PROCESSED
            self.uow_dao.update(uow)

            self.logger.info('Completed %s with result = %r' % (self.thread_name, self.return_code))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.thread_name, exc_info=True)
            uow.state = unit_of_work.STATE_INVALID
            self.uow_dao.update(uow)
        finally:
            self.logger.info('}')
            self.alive = False

    def run(self):
        try:
            self._start_process()
            code = None
            alive = True
            while alive:
                alive, code = self._poll_process()
                time.sleep(0.1)

            if code == 0:
                self.performance_ticker.tracker.increment_success()
            else:
                self.performance_ticker.tracker.increment_failure()

            self.logger.info('BashDriver for %s return code is %r' % (self.thread_name, code))
        except Exception as e:
            self.performance_ticker.tracker.increment_failure()
            self.logger.error('Safety fuse while processing request %r: %r' % (self.message.body, e), exc_info=True)
        finally:
            self.consumer.acknowledge(self.message.delivery_tag)