Python IBackend.IBackend类代码示例

    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('ARC bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick

    def master_kill(self):
        '''kill the master job to the grid'''

        job = self.getJobObject()

        if not job.master and len(job.subjobs) == 0:
            return IBackend.master_kill(self)
        elif job.master:
            return IBackend.master_kill(self)
        else:
            return self.master_bulk_kill()

    def __mt_job_prepare__(self, rjobs, subjobconfigs, masterjobconfig):
        '''preparing jobs in multiple threads'''

        logger.warning(
            'preparing %d subjobs ... it may take a while' % len(rjobs))

        # prepare the master job (i.e. create shared inputsandbox, etc.)
        master_input_sandbox = IBackend.master_prepare(self, masterjobconfig)

        # uploading the master job if it's over the WMS sandbox limitation
        for f in master_input_sandbox:
            master_input_idx = self.__check_and_prestage_inputfile__(f)

            if not master_input_idx:
                logger.error('master input sandbox perparation failed: %s' % f)
                return None

        # the algorithm for preparing a single bulk job
        class MyAlgorithm(Algorithm):

            def __init__(self):
                Algorithm.__init__(self)

            def process(self, sj_info):
                my_sc = sj_info[0]
                my_sj = sj_info[1]

                try:
                    logger.debug("preparing job %s" % my_sj.getFQID('.'))
                    jdlpath = my_sj.backend.preparejob(
                        my_sc, master_input_sandbox)

                    if (not jdlpath) or (not os.path.exists(jdlpath)):
                        raise GangaException(
                            'job %s not properly prepared' % my_sj.getFQID('.'))

                    self.__appendResult__(my_sj.id, jdlpath)
                    return True
                except Exception as x:
                    log_user_exception()
                    return False

        mt_data = []
        for sc, sj in zip(subjobconfigs, rjobs):
            mt_data.append([sc, sj])

        myAlg = MyAlgorithm()
        myData = Data(collection=mt_data)

        runner = MTRunner(
            name='lcg_jprepare', algorithm=myAlg, data=myData, numThread=10)
        runner.start()
        runner.join(-1)

        if len(runner.getDoneList()) < len(mt_data):
            return None
        else:
            # return a JDL file dictionary with subjob ids as keys, JDL file
            # paths as values
            return runner.getResults()

    def master_resubmit(self, rjobs):
        '''Resubmit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        ick = False

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug('rjobs: %s' % str(rjobs))
            logger.debug('mode: master job normal resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug('mode: individual subjob resubmission')
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug('mode: master job resubmission')

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException('CREAM bulk submission failure')

        profiler.check('job re-submission elapsed time')

        return ick

    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        '''Submit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        # finding ARC CE endpoint for job submission
        #allowed_celist = []
        # try:
        #    allowed_celist = self.requirements.getce()
        #    if not self.CE and allowed_celist:
        #        self.CE = allowed_celist[0]
        # except:
        #    logger.warning('ARC CE assigment from ARCRequirements failed.')

        # if self.CE and allowed_celist:
        #    if self.CE not in allowed_celist:
        #        logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0] ) )
        #        self.CE = allowed_celist[0]

        # use arc info to check for any endpoints recorded in the config file
        rc, output = grids['GLITE'].arc_info()

        if not self.CE and rc != 0:
            raise GangaException(
                "ARC CE endpoint not set and no default settings in '%s'. " % config['ArcConfigFile'])
        elif self.CE:
            logger.info('ARC CE endpoint set to: ' + str(self.CE))
        else:
            logger.info("Using ARC CE endpoints defined in '%s'" %
                        config['ArcConfigFile'])

        # delegate proxy to ARC CE
        # if not grids['GLITE'].arc_proxy_delegation(self.CE):
        #    logger.warning('proxy delegation to %s failed' % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(
                self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(
                rjobs, subjobconfigs, masterjobconfig)

        profiler.check('==> master_submit() elapsed time')

        return ick

    def master_resubmit(self, rjobs):
        """Resubmit the master job to the grid"""

        profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
        profiler.start()

        job = self.getJobObject()

        ick = False

        # delegate proxy to CREAM CE
        if not grids["GLITE"].cream_proxy_delegation(self.CE):
            logger.warning("proxy delegation to %s failed" % self.CE)

        if not job.master and len(job.subjobs) == 0:
            # case 1: master job normal resubmission
            logger.debug("rjobs: %s" % str(rjobs))
            logger.debug("mode: master job normal resubmission")
            ick = IBackend.master_resubmit(self, rjobs)

        elif job.master:
            # case 2: individual subjob resubmission
            logger.debug("mode: individual subjob resubmission")
            ick = IBackend.master_resubmit(self, rjobs)

        else:
            # case 3: master job bulk resubmission
            logger.debug("mode: master job resubmission")

            ick = self.master_bulk_resubmit(rjobs)
            if not ick:
                raise GangaException("CREAM bulk submission failure")

        profiler.check("job re-submission elapsed time")

        return ick

    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        '''Submit the master job to the grid'''

        profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
        profiler.start()

        job = self.getJobObject()

        # finding CREAM CE endpoint for job submission
        allowed_celist = []
        try:
            allowed_celist = self.requirements.getce()
            if not self.CE and allowed_celist:
                self.CE = allowed_celist[0]
        except:
            logger.warning(
                'CREAM CE assigment from AtlasCREAMRequirements failed.')

        if self.CE and allowed_celist:
            if self.CE not in allowed_celist:
                logger.warning('submission to CE not allowed: %s, use %s instead' % (
                    self.CE, allowed_celist[0]))
                self.CE = allowed_celist[0]

        if not self.CE:
            raise GangaException('CREAM CE endpoint not set')

        # delegate proxy to CREAM CE
        self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
        if not self.delegation_id:
            logger.warning('proxy delegation to %s failed' % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(
                self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(
                rjobs, subjobconfigs, masterjobconfig)

        profiler.check('==> master_submit() elapsed time')

        return ick

    def __mt_job_prepare__(self, rjobs, subjobconfigs, masterjobconfig):
        '''preparing jobs in multiple threads'''

        logger.warning('preparing %d subjobs ... it may take a while' % len(rjobs))

        # prepare the master job (i.e. create shared inputsandbox, etc.)
        master_input_sandbox=IBackend.master_prepare(self,masterjobconfig)

        ## uploading the master job if it's over the WMS sandbox limitation
        for f in master_input_sandbox:
            master_input_idx = self.__check_and_prestage_inputfile__(f)

            if not master_input_idx:
                logger.error('master input sandbox perparation failed: %s' % f)
                return None

        # the algorithm for preparing a single bulk job
        class MyAlgorithm(Algorithm):

            def __init__(self):
                Algorithm.__init__(self)

            def process(self, sj_info):
                my_sc = sj_info[0]
                my_sj = sj_info[1]

                try:
                    logger.debug("preparing job %s" % my_sj.getFQID('.'))
                    jdlpath = my_sj.backend.preparejob(my_sc, master_input_sandbox)

                    if (not jdlpath) or (not os.path.exists(jdlpath)):
                        raise GangaException('job %s not properly prepared' % my_sj.getFQID('.'))

                    self.__appendResult__( my_sj.id, jdlpath )
                    return True
                except Exception,x:
                    log_user_exception()
                    return False

    def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
        """Submit the master job to the grid"""

        profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
        profiler.start()

        job = self.getJobObject()

        # finding CREAM CE endpoint for job submission
        allowed_celist = []
        try:
            allowed_celist = self.requirements.getce()
            if not self.CE and allowed_celist:
                self.CE = allowed_celist[0]
        except:
            logger.warning("CREAM CE assigment from AtlasCREAMRequirements failed.")

        if self.CE and allowed_celist:
            if self.CE not in allowed_celist:
                logger.warning("submission to CE not allowed: %s, use %s instead" % (self.CE, allowed_celist[0]))
                self.CE = allowed_celist[0]

        if not self.CE:
            raise GangaException("CREAM CE endpoint not set")

        # delegate proxy to CREAM CE
        if not grids["GLITE"].cream_proxy_delegation(self.CE):
            logger.warning("proxy delegation to %s failed" % self.CE)

        # doing massive job preparation
        if len(job.subjobs) == 0:
            ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig)
        else:
            ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig)

        profiler.check("==> master_submit() elapsed time")

        return ick

 def master_submit(self,rjobs,subjobconfigs,masterjobconfig):
     #if self.error.method == 'master_submit':
     self.error.trigger()
     return IBackend.master_submit(self,rjobs,subjobconfigs,masterjobconfig)

 def master_submit(self, rjobs, subjobconfigs, masterjobconfig, keep_going=False):
     """ Overload master_submit to avoid parallel submission with Interactive backend"""
     return IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig, keep_going)