Python process_context.ProcessContext类代码示例

    def entries(self):
        list_of_trees = []
        try:
            sorter_keys = sorted(context.timetable_context.keys())
            for tree_name in sorter_keys:
                tree_obj = self.mbean.timetable.trees[tree_name]

                tree_row = list()
                tree_row.append(tree_name)                                              # index 0
                tree_row.append(tree_obj.mx_page)                                       # index 1
                tree_row.append(tree_obj.mx_name)                                       # index 2

                processes = dict()                                                      # index 3
                context_entry = context.timetable_context[tree_name]
                for process_name in context_entry.enclosed_processes:
                    process_details = [process_name,                                                 # index x0
                                       ProcessContext.get_time_qualifier(process_name),              # index x1
                                       self._state_machine_name(process_name),                       # index x2
                                       ProcessContext.get_process_type(process_name),                # index x3
                                       ProcessContext.run_on_active_timeperiod(process_name),        # index x4
                                       context_entry.dependent_on,                                   # index x5
                                       self._list_of_dependant_trees(tree_obj)]                      # index x6
                    processes[process_name] = process_details
                tree_row.append(processes)

                list_of_trees.append(tree_row)
        except Exception as e:
            self.logger.error('MX Exception %s' % str(e), exc_info=True)

        return list_of_trees

def stop_process(options):
    """Stop the synergy-data daemons"""
    import logging
    from supervisor import supervisor_helper as helper
    from model.box_configuration_entry import BoxConfigurationEntry

    if options.app is not None and options.app != process_context.PROCESS_SUPERVISOR:
        # mark individual process for termination
        # real work is performed by Supervisor
        if options.app not in PROCESSES_FOR_XXL:
            sys.stdout.write("ERROR: requested process must be withing allowed list of: %r \n" % PROCESSES_FOR_XXL)
            sys.exit(1)

        box_id = helper.get_box_id(logging)
        box_configuration = helper.retrieve_configuration(logging, box_id)
        box_configuration.set_process_state(options.app, BoxConfigurationEntry.STATE_OFF)
        helper.update_configuration(logging, box_configuration)
    else:
        # stop Supervisor
        try:
            pid = _get_supervisor_pid()
            if pid is None:
                message = "ERROR: Can not find Supervisor pidfile. Supervisor not running?\n"
                sys.stderr.write(message)
                sys.exit(1)

            # Try killing the daemon process
            sys.stdout.write("INFO: Killing %r \n" % process_context.PROCESS_SUPERVISOR)
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)
            ProcessContext.remove_pid_file(process_context.PROCESS_SUPERVISOR)
        except Exception as e:
            sys.stderr.write("Exception on killing %s : %s \n" % (process_context.PROCESS_SUPERVISOR, str(e)))
            sys.exit(0)

def create_unit_of_work(process_name,
                        start_id,
                        end_id,
                        timeperiod='INVALID_TIMEPERIOD',
                        state=unit_of_work.STATE_REQUESTED,
                        creation_at=datetime.utcnow(),
                        uow_id=None):
    """ method creates and returns unit_of_work """
    try:
        source_collection = ProcessContext.get_source(process_name)
        target_collection = ProcessContext.get_sink(process_name)
    except KeyError:
        source_collection = None
        target_collection = None

    uow = UnitOfWork()
    uow.timeperiod = timeperiod
    uow.start_timeperiod = timeperiod
    uow.end_timeperiod = timeperiod
    uow.start_id = start_id
    uow.end_id = end_id
    uow.source = source_collection
    uow.sink = target_collection
    uow.state = state
    uow.created_at = creation_at
    uow.process_name = process_name
    uow.number_of_retries = 0

    if uow_id is not None:
        uow.document['_id'] = uow_id

    return uow

    def compute_scope_of_processing(self, process_name, start_timeperiod, end_timeperiod, job_record):
        """method reads collection and identify slice for processing"""
        source_collection_name = ProcessContext.get_source(process_name)
        target_collection_name = ProcessContext.get_sink(process_name)

        start_id = self.ds.highest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)
        end_id = self.ds.lowest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)

        uow = UnitOfWork()
        uow.timeperiod = start_timeperiod
        uow.start_id = str(start_id)
        uow.end_id = str(end_id)
        uow.start_timeperiod = start_timeperiod
        uow.end_timeperiod = end_timeperiod
        uow.created_at = datetime.utcnow()
        uow.source = source_collection_name
        uow.sink = target_collection_name
        uow.state = unit_of_work.STATE_REQUESTED
        uow.process_name = process_name
        uow.number_of_retries = 0
        uow_id = self.uow_dao.insert(uow)

        mq_request = WorkerMqRequest()
        mq_request.process_name = process_name
        mq_request.unit_of_work_id = uow_id

        publisher = self.publishers.get(process_name)
        publisher.publish(mq_request.document)
        publisher.release()

        msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_timeperiod)
        self._log_message(INFO, process_name, job_record, msg)
        return uow

 def test_double_initialization(self):
     try:
         ProcessContext.set_current_process(PROCESS_GC)
         ProcessContext.set_current_process(PROCESS_SUPERVISOR)
         self.assertTrue(False, 'set_current_process should not allow double initialization')
     except AttributeError:
         self.assertTrue(True)

    def __init__(self, process_name):
        """ renames process to SynergyYYY and creates PID file """
        self.process_name = process_name
        self.logger = ProcessContext.get_logger(process_name)

        # process-related activities
        setproctitle.setproctitle(settings['process_prefix'] + self.process_name)
        ProcessContext.create_pid_file(self.process_name)

def run(process_name):
    """
    You should override this method when you subclass Daemon. It will be
    called after the process has been daemonized by start() or restart().
    """

    sys.stdout.write('INFO: Starting %s \n' % ProcessContext.get_classname(process_name))
    klass = get_class(ProcessContext.get_classname(process_name))
    instance = klass(process_name)
    instance.start()

    def __init__(self, process_name, process_id=None):
        """ renames process to SynergyYYY and creates PID file """
        self.process_name = process_name
        self.process_id = process_id
        self.logger = ProcessContext.get_logger(process_name, process_id=self.process_id)

        # process-related activities
        process_title = settings['process_prefix'] + self.process_name
        if self.process_id:
            process_title += str(self.process_id)
        setproctitle.setproctitle(process_title)
        ProcessContext.create_pid_file(self.process_name, process_id=self.process_id)

def kill_process(process_name):
    """ method is called to kill a running process """
    try:
        sys.stdout.write('killing: %s { \n' % process_name)
        pid = get_process_pid(process_name)
        if pid is not None and psutil.pid_exists(int(pid)):
            p = psutil.Process(pid)
            p.kill()
            p.wait()
            ProcessContext.remove_pid_file(process_name)
    except Exception as e:
        sys.stderr.write('Exception on killing %s : %s \n' % (process_name, str(e)))
    finally:
        sys.stdout.write('}')

    def __init__(self, process_name):
        """@param process_name: id of the process, the worker will be performing """
        super(AbstractMqWorker, self).__init__(process_name)
        self.queue_source = ProcessContext.get_source(self.process_name)
        self.queue_sink = ProcessContext.get_sink(self.process_name)
        self.consumer = None
        self._init_mq_consumer()

        self.main_thread = None
        self.performance_ticker = None
        self._init_performance_ticker(self.logger)

        msg_suffix = 'in Production Mode'
        if settings['under_test']:
            msg_suffix = 'in Testing Mode'
        self.logger.info('Started %s %s' % (self.process_name, msg_suffix))

    def get_reprocessing_candidates(self, since=None):
        """ method queries Unit Of Work whose <start_timeperiod> is younger than <since>
        and who could be candidates for re-processing """
        collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)

        query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
                                              unit_of_work.STATE_INVALID,
                                              unit_of_work.STATE_REQUESTED]}}

        if since is None:
            cursor = collection.find(query).sort('_id', ASCENDING)
            candidates = [UnitOfWork(document) for document in cursor]
        else:
            candidates = []
            yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
            query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}

            cursor = collection.find(query).sort('_id', ASCENDING)
            for document in cursor:
                uow = UnitOfWork(document)
                if uow.process_name not in ProcessContext.CONTEXT:
                    # this is a decommissioned process
                    continue

                time_qualifier = ProcessContext.get_time_qualifier(uow.process_name)
                if time_qualifier == QUALIFIER_REAL_TIME:
                    time_qualifier = QUALIFIER_HOURLY
                process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)

                if process_specific_since <= uow.start_timeperiod:
                    candidates.append(uow)

        if len(candidates) == 0:
            raise LookupError('MongoDB has no reprocessing candidates units of work')
        return candidates

    def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
        """ method that takes care of processing job records in STATE_IN_PROGRESS state"""
        time_qualifier = ProcessContext.get_time_qualifier(process_name)
        end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
        actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
        can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
        uow = self.uow_dao.get_one(job_record.related_unit_of_work)

        if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
            if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]:
                # current uow has not been processed yet. update it
                self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record)
            else:
                # cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED
                # create new uow to cover new inserts
                self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)

        elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
            # create new uow for FINAL RUN
            self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record)

        else:
            msg = 'job record %s has timeperiod from future %s vs current time %s' \
                  % (job_record.document['_id'], start_timeperiod, actual_timeperiod)
            self._log_message(ERROR, process_name, job_record, msg)

    def _start_process(self, start_timeperiod, end_timeperiod, arguments):
        try:
            start_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, start_timeperiod)
            sqoop_slice_starttime = start_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)

            end_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, end_timeperiod)
            sqoop_slice_endtime = end_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)

            sink_path = ProcessContext.get_sink(self.process_name)

            self.logger.info('start: %s {' % self.process_name)
            p = psutil.Popen([settings['bash_shell'],
                              settings['sqoop_command'],
                              str(sqoop_slice_starttime),
                              str(sqoop_slice_endtime),
                              sink_path + '/' + start_timeperiod],
                             close_fds=True,
                             cwd=settings['process_cwd'],
                             stdin=PIPE,
                             stdout=PIPE,
                             stderr=PIPE)
            self.cli_process = p
            self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid))
        except Exception:
            self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True)
        finally:
            self.logger.info('}')

    def _load_managed_entries(self):
        """ loads scheduler managed entries. no start-up procedures are performed """
        scheduler_entries = self.se_managed_dao.get_all()
        for scheduler_entry_obj in scheduler_entries:
            process_name = scheduler_entry_obj.process_name
            if scheduler_entry_obj.process_name not in ProcessContext.CONTEXT:
                self.logger.error('Process %r is not known to the system. Skipping it.' % process_name)
                continue

            process_type = ProcessContext.get_process_type(process_name)
            if process_type in [TYPE_BLOCKING_DEPENDENCIES, TYPE_BLOCKING_CHILDREN, TYPE_MANAGED]:
                function = self.fire_managed_worker
                handler_type = TYPE_MANAGED
            elif process_type == TYPE_GARBAGE_COLLECTOR:
                function = self.fire_garbage_collector
                handler_type = TYPE_MANAGED
            elif process_type == TYPE_FREERUN:
                self.logger.error('TYPE_FREERUN process %s was found in scheduler_managed_entry table. '
                                  'Move the process to the scheduler_freerun_entry table. Skipping the process.'
                                  % process_type)
                continue
            else:
                self.logger.error('Process type %s is not known to the system. Skipping it.' % process_type)
                continue

            self._activate_handler(scheduler_entry_obj, process_name, 'NA', function, handler_type)

def query_configuration(options):
    """ Queries process state """
    from system import process_helper

    if not options.supervisor:
        # reads status of one process only
        process_helper.poll_process(options.app)
    else:
        # reads current box configuration and prints it to the console
        from db.dao.box_configuration_dao import BoxConfigurationDao
        from supervisor import supervisor_helper as helper
        from system.process_context import ProcessContext
        from constants import PROCESS_LAUNCH_PY

        logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
        box_id = helper.get_box_id(logger)
        bc_dao = BoxConfigurationDao(logger)
        sys.stdout.write('\nConfiguration for BOX_ID=%r:\n' % box_id)
        box_configuration = bc_dao.get_one(box_id)
        process_list = box_configuration.get_process_list()
        i = 1
        for process in process_list:
            sys.stdout.write('%d\t%r:%r \n' % (i, process, process_list[process]))
            i += 1
        sys.stdout.write('\n')