Python ProcessContext.get_time_qualifier方法代码示例

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
        """ method that takes care of processing job records in STATE_IN_PROGRESS state"""
        time_qualifier = ProcessContext.get_time_qualifier(process_name)
        end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
        actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
        can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
        uow = self.uow_dao.get_one(job_record.related_unit_of_work)

        if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
            if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]:
                # current uow has not been processed yet. update it
                self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record)
            else:
                # cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED
                # create new uow to cover new inserts
                self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)

        elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
            # create new uow for FINAL RUN
            self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record)

        else:
            msg = 'job record %s has timeperiod from future %s vs current time %s' \
                  % (job_record.document['_id'], start_timeperiod, actual_timeperiod)
            self._log_message(ERROR, process_name, job_record, msg)

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
def cast_to_time_qualifier(process_name, timestamp):
    """ method is used to cast synergy_time accordingly to process time qualifier.
    For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """
    
    date_format = None
    qualifier = ProcessContext.get_time_qualifier(process_name)
    if qualifier == ProcessContext.QUALIFIER_HOURLY:
        if len(timestamp) > 10:
            t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN)
            return t.strftime(SYNERGY_DATE_PATTERN)
        else:
            return timestamp

    elif qualifier == ProcessContext.QUALIFIER_DAILY :
        date_format = '%Y%m%d00'
    elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'

    pattern = define_pattern(timestamp)
    t = datetime.strptime(timestamp, pattern)

    if date_format is not None:
        return t.strftime(date_format)
    else:
        raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def entries(self):
        list_of_trees = []
        try:
            sorter_keys = sorted(context.timetable_context.keys())
            for tree_name in sorter_keys:
                tree_obj = self.mbean.timetable.trees[tree_name]

                tree_row = list()
                tree_row.append(tree_name)                                              # index 0
                tree_row.append(tree_obj.mx_page)                                       # index 1
                tree_row.append(tree_obj.mx_name)                                       # index 2

                processes = dict()                                                      # index 3
                context_entry = context.timetable_context[tree_name]
                for process_name in context_entry.enclosed_processes:
                    process_details = [process_name,                                                 # index x0
                                       ProcessContext.get_time_qualifier(process_name),              # index x1
                                       self._state_machine_name(process_name),                       # index x2
                                       ProcessContext.get_process_type(process_name),                # index x3
                                       ProcessContext.run_on_active_timeperiod(process_name),        # index x4
                                       context_entry.dependent_on,                                   # index x5
                                       self._list_of_dependant_trees(tree_obj)]                      # index x6
                    processes[process_name] = process_details
                tree_row.append(processes)

                list_of_trees.append(tree_row)
        except Exception as e:
            self.logger.error('MX Exception %s' % str(e), exc_info=True)

        return list_of_trees

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def get_reprocessing_candidates(self, since=None):
        """ method queries Unit Of Work whose <start_timeperiod> is younger than <since>
        and who could be candidates for re-processing """
        collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)

        query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
                                              unit_of_work.STATE_INVALID,
                                              unit_of_work.STATE_REQUESTED]}}

        if since is None:
            cursor = collection.find(query).sort('_id', ASCENDING)
            candidates = [UnitOfWork(document) for document in cursor]
        else:
            candidates = []
            yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
            query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}

            cursor = collection.find(query).sort('_id', ASCENDING)
            for document in cursor:
                uow = UnitOfWork(document)
                if uow.process_name not in ProcessContext.CONTEXT:
                    # this is a decommissioned process
                    continue

                time_qualifier = ProcessContext.get_time_qualifier(uow.process_name)
                if time_qualifier == QUALIFIER_REAL_TIME:
                    time_qualifier = QUALIFIER_HOURLY
                process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)

                if process_specific_since <= uow.start_timeperiod:
                    candidates.append(uow)

        if len(candidates) == 0:
            raise LookupError('MongoDB has no reprocessing candidates units of work')
        return candidates

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
def increment_time(process_name, timestamp):
    """ method is used by Scheduler to define <<next>> time period.
    For hourly, it is next hour: 20100101_19 -> 20100101_20 
    For month - next month: 201001 -> 201002, etc"""
    
    qualifier = ProcessContext.get_time_qualifier(process_name)

    pattern = define_pattern(timestamp)
    t = datetime.strptime(timestamp, pattern)

    if qualifier == ProcessContext.QUALIFIER_HOURLY:
        t = t + timedelta(hours=1)
        return t.strftime('%Y%m%d%H')
    
    elif qualifier == ProcessContext.QUALIFIER_DAILY:
        t = t + timedelta(days=1)
        return t.strftime('%Y%m%d00')
    
    elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
        if t.month + 1 > 12:
            new_month = 1
            new_year = t.year + 1
            t = t.replace(year = new_year, month = new_month)
        else:
            t = t.replace(month = t.month + 1)
        return t.strftime('%Y%m0000')
    
    elif qualifier == ProcessContext.QUALIFIER_YEARLY:
        t = t.replace(year = t.year + 1)
        return t.strftime('%Y000000')
    else:
        raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))        

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def details(self):
        resp = dict()
        timetable = self.mbean.timetable
        tree = timetable.get_tree(self.process_name)

        if self.timeperiod is None and tree is not None:
            # return list of yearly nodes OR leafs for linear tree
            resp['children'] = dict()

            # limit number of children to return, since linear tree can holds thousands of nodes
            sorted_keys = sorted(tree.root.children.keys(), reverse=True)
            sorted_keys = sorted_keys[:settings['mx_children_limit']]
            for key in sorted_keys:
                child = tree.root.children[key]
                resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)

        elif tree is not None:
            time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
            self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
            node = tree.get_node_by_process(self.process_name, self.timeperiod)
            resp['node'] = TreeNodeDetails.get_details(self.logger, node)
            resp['children'] = dict()
            for key in node.children:
                child = node.children[key]
                resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)

        return resp

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
        """ method that takes care of processing job records in STATE_IN_PROGRESS state"""
        time_qualifier = ProcessContext.get_time_qualifier(process_name)
        end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
        actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
        can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
        uow = self.uow_dao.get_one(job_record.related_unit_of_work)
        iteration = int(uow.end_id)

        try:
            if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
                if uow.state in [unit_of_work.STATE_REQUESTED,
                                 unit_of_work.STATE_IN_PROGRESS,
                                 unit_of_work.STATE_INVALID]:
                    # Large Job processing takes more than 1 tick of Scheduler
                    # Let the Large Job processing complete - do no updates to Scheduler records
                    pass
                elif uow.state in [unit_of_work.STATE_PROCESSED,
                                   unit_of_work.STATE_CANCELED]:
                    # create new uow to cover new inserts
                    uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, iteration + 1, job_record)
                    self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS)

            elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
                if uow.state in [unit_of_work.STATE_REQUESTED,
                                 unit_of_work.STATE_IN_PROGRESS,
                                 unit_of_work.STATE_INVALID]:
                    # Job processing has not started yet
                    # Let the processing complete - do no updates to Scheduler records
                    msg = 'Suppressed creating uow for %s in timeperiod %s; job record is in %s; uow is in %s' \
                          % (process_name, job_record.timeperiod, job_record.state, uow.state)
                elif uow.state == unit_of_work.STATE_PROCESSED:
                    self.timetable.update_job_record(process_name, job_record, uow, job.STATE_PROCESSED)
                    timetable_tree = self.timetable.get_tree(process_name)
                    timetable_tree.build_tree()
                    msg = 'Transferred job record %s in timeperiod %s to STATE_PROCESSED for %s' \
                          % (job_record.document['_id'], job_record.timeperiod, process_name)
                elif uow.state == unit_of_work.STATE_CANCELED:
                    self.timetable.update_job_record(process_name, job_record, uow, job.STATE_SKIPPED)
                    msg = 'Transferred job record %s in timeperiod %s to STATE_SKIPPED for %s' \
                          % (job_record.document['_id'], job_record.timeperiod, process_name)
                else:
                    msg = 'Unknown state %s for job record %s in timeperiod %s for %s' \
                          % (uow.state, job_record.document['_id'], job_record.timeperiod, process_name)

                self._log_message(INFO, process_name, job_record, msg)
            else:
                msg = 'Job record %s has timeperiod from future %s vs current time %s' \
                      % (job_record.document['_id'], start_timeperiod, actual_timeperiod)
                self._log_message(ERROR, process_name, job_record, msg)

        except DuplicateKeyError as e:
            uow = self.recover_from_duplicatekeyerror(e)
            if uow is not None:
                self.timetable.update_job_record(process_name, job_record, uow, job_record.state)
            else:
                msg = 'MANUAL INTERVENTION REQUIRED! Unable to identify unit_of_work for %s in %s' \
                      % (process_name, job_record.timeperiod)
                self._log_message(ERROR, process_name, job_record, msg)

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def action_get_log(self):
        resp = dict()
        timetable = self.mbean.timetable
        tree = timetable.get_tree(self.process_name)

        if tree is not None:
            time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
            self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
            node = tree.get_node_by_process(self.process_name, self.timeperiod)
            resp['log'] = node.job_record.log

        return resp

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
 def __init__(self, tree, parent, process_name, timeperiod, job_record):
     # initializes the data members
     self.children = dict()
     self.tree = tree
     self.parent = parent
     self.process_name = process_name
     self.timeperiod = timeperiod
     self.job_record = job_record
     if parent is None and process_name is None and timeperiod is None and job_record is None:
         # special case - node is TREE ROOT
         self.time_qualifier = None
     else:
         self.time_qualifier = ProcessContext.get_time_qualifier(self.process_name)

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
 def _get_timetable_collection(self, process_name):
     """timetable stores timeperiod in 4 collections: hourly, daily, monthly and yearly; method looks for the
     proper timetable_collection base on process TIME_QUALIFIER"""
     qualifier = ProcessContext.get_time_qualifier(process_name)
     if qualifier == ProcessContext.QUALIFIER_HOURLY:
         collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_HOURLY)
     elif qualifier == ProcessContext.QUALIFIER_DAILY:
         collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_DAILY)
     elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
         collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_MONTHLY)
     elif qualifier == ProcessContext.QUALIFIER_YEARLY:
         collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_YEARLY)
     else:
         raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
     return collection

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def _get_job_collection(self, process_name):
        """jobs are stored in 4 collections: hourly, daily, monthly and yearly;
        method looks for the proper job_collection base on process TIME_QUALIFIER"""
        qualifier = ProcessContext.get_time_qualifier(process_name)

        if qualifier == QUALIFIER_HOURLY:
            collection = self.ds.connection(COLLECTION_JOB_HOURLY)
        elif qualifier == QUALIFIER_DAILY:
            collection = self.ds.connection(COLLECTION_JOB_DAILY)
        elif qualifier == QUALIFIER_MONTHLY:
            collection = self.ds.connection(COLLECTION_JOB_MONTHLY)
        elif qualifier == QUALIFIER_YEARLY:
            collection = self.ds.connection(COLLECTION_JOB_YEARLY)
        else:
            raise ValueError('Unknown time qualifier: %s for %s' % (qualifier, process_name))
        return collection

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def action_skip(self):
        resp = dict()
        timetable = self.mbean.timetable
        tree = timetable.get_tree(self.process_name)

        if tree is not None:
            time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
            self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
            node = tree.get_node_by_process(self.process_name, self.timeperiod)
            self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name))
            effected_nodes = node.request_skip()
            for node in effected_nodes:
                resp[node.timeperiod] = TreeNodeDetails.get_details(self.logger, node)
            self.logger.info('}')

        return resp

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
def synergy_to_datetime(process_name, timestamp):
    """ method receives timestamp in Synergy format YYYYMMDD_HH and convert it to _naive_ datetime"""
    qualifier = ProcessContext.get_time_qualifier(process_name)
    if qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
    return datetime.strptime(timestamp, date_format).replace(tzinfo=None)

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
def datetime_to_synergy(process_name, dt):
    """ method parses datetime and returns Synergy Date"""
    date_format = None
    qualifier = ProcessContext.get_time_qualifier(process_name)
    if qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))        
    return dt.strftime(date_format)

# 需要导入模块: from system.process_context import ProcessContext [as 别名]
# 或者: from system.process_context.ProcessContext import get_time_qualifier [as 别名]
    def _process_state_embryo(self, process_name, job_record, start_timeperiod):
        """ method that takes care of processing job records in STATE_EMBRYO state"""
        time_qualifier = ProcessContext.get_time_qualifier(process_name)
        end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)

        try:
            uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, 0, job_record)
        except DuplicateKeyError as e:
            msg = 'Catching up with latest unit_of_work %s in timeperiod %s, because of: %r' \
                  % (process_name, job_record.timeperiod, e)
            self._log_message(WARNING, process_name, job_record, msg)
            uow = self.recover_from_duplicatekeyerror(e)

        if uow is not None:
            self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS)
        else:
            msg = 'MANUAL INTERVENTION REQUIRED! Unable to locate unit_of_work for %s in %s' \
                  % (process_name, job_record.timeperiod)
            self._log_message(WARNING, process_name, job_record, msg)