Python Proxy.Proxy类代码示例

 def execute(self, *args, **kwargs):
     result = None
     proxycfg = {'vo': kwargs['task']['tm_user_vo'],
                 'logger': self.logger,
                 'myProxySvr': self.config.Services.MyProxy,
                 'proxyValidity' : '144:0',
                 'min_time_left' : 36000, ## do we need this ? or should we use self.myproxylen? 
                 'userDN' : kwargs['task']['tm_user_dn'],
                 'group' : kwargs['task']['tm_user_group'] if kwargs['task']['tm_user_group'] else '',
                 'role' : kwargs['task']['tm_user_role'] if kwargs['task']['tm_user_role'] else '',
                 'server_key': self.config.MyProxy.serverhostkey,
                 'server_cert': self.config.MyProxy.serverhostcert,
                 'serverDN': self.config.MyProxy.serverdn,
                 'uisource': getattr(self.config.MyProxy, 'uisource', ''),
                 'credServerPath': self.config.MyProxy.credpath,
                 'myproxyAccount' : self.server['host'],
                 'cleanEnvironment' : getattr(self.config.MyProxy, 'cleanEnvironment', False)
                }
     proxy = Proxy(proxycfg)
     userproxy = proxy.getProxyFilename(serverRenewer=True)
     proxy.logonRenewMyProxy()
     timeleft = proxy.getTimeLeft(userproxy)
     if timeleft is None or timeleft <= 0:
         msg = "Impossible to retrieve proxy from %s for %s." % (proxycfg['myProxySvr'], proxycfg['userDN'])
         raise TaskWorkerException(msg)
     else:
         kwargs['task']['user_proxy'] = userproxy
         result = Result(task=kwargs['task'], result='OK')
     return result

 def get_proxy(self, ad):
     result = None
     vo = 'cms'
     group = ''
     role = ''
     if 'CRAB_UserVO' in ad and ad['CRAB_UserVO']:
         vo = ad['CRAB_UserVO']
     if 'CRAB_UserGroup' in ad and ad['CRAB_UserGroup'] and ad['CRAB_UserGroup'] != classad.Value.Undefined:
         group = ad['CRAB_UserGroup']
     if 'CRAB_UserRole' in ad and ad['CRAB_UserRole'] and ad['CRAB_UserRole'] != classad.Value.Undefined:
         role = ad['CRAB_UserRole']
     proxycfg = {'vo': vo,
                 'logger': self.logger,
                 'myProxySvr': self.config.Services.MyProxy,
                 'myproxyAccount': self.config.TaskWorker.resturl,
                 'proxyValidity' : '144:0',
                 'min_time_left' : MINPROXYLENGTH, ## do we need this ? or should we use self.myproxylen? 
                 'userDN' : ad['CRAB_UserDN'],
                 'group' : group,
                 'role' : role,
                 'server_key': self.config.MyProxy.serverhostkey,
                 'server_cert': self.config.MyProxy.serverhostcert,
                 'serverDN': self.config.MyProxy.serverdn,
                 'uisource': getattr(self.config.MyProxy, 'uisource', ''),
                 'credServerPath': self.config.MyProxy.credpath,
                 'cleanEnvironment' : getattr(self.config.MyProxy, 'cleanEnvironment', False)}
     proxy = Proxy(proxycfg)
     userproxy = proxy.getProxyFilename(serverRenewer=True)
     proxy.logonRenewMyProxy()
     timeleft = proxy.getTimeLeft(userproxy)
     if timeleft is None or timeleft <= 0:
         self.logger.error("Impossible to retrieve proxy from %s for %s." %(proxycfg['myProxySvr'], proxycfg['userDN']))
         raise Exception("Failed to retrieve proxy.")
     return userproxy

    def query_database(self):
        cred = Proxy({'logger': logging.getLogger("WMCore")})
        dbs = DASWrapper(self.dbs_instance, ca_info=cred.getProxyFilename())

        baseinfo = dbs.listFileSummaries(dataset=self.dataset)
        if baseinfo is None or (len(baseinfo) == 1 and baseinfo[0] is None):
            raise ValueError('unable to retrive information for dataset {}'.format(self.dataset))

        if not self.file_based:
            result = self.__cache.cached(self.dataset, self.lumi_mask, baseinfo)
            if result:
                return result
        total_lumis = sum([info['num_lumi'] for info in baseinfo])

        result = DatasetInfo()
        result.total_events = sum([info['num_event'] for info in baseinfo])

        for info in dbs.listFiles(dataset=self.dataset, detail=True):
            fn = info['logical_file_name']
            result.files[fn].events = info['event_count']
            result.files[fn].size = info['file_size']

        if self.file_based:
            for info in dbs.listFiles(dataset=self.dataset):
                fn = info['logical_file_name']
                result.files[fn].lumis = [(-2, -2)]
        else:
            blocks = dbs.listBlocks(dataset=self.dataset)
            if self.lumi_mask:
                unmasked_lumis = LumiList(filename=self.lumi_mask)
            for block in blocks:
                runs = dbs.listFileLumis(block_name=block['block_name'])
                for run in runs:
                    fn = run['logical_file_name']
                    for lumi in run['lumi_section_num']:
                        if not self.lumi_mask or ((run['run_num'], lumi) in unmasked_lumis):
                            result.files[fn].lumis.append((run['run_num'], lumi))
                        elif self.lumi_mask and ((run['run_num'], lumi) not in unmasked_lumis):
                            result.masked_units += 1

        result.unmasked_units = sum([len(f.lumis) for f in result.files.values()])
        result.total_units = result.unmasked_units + result.masked_units

        if not self.file_based:
            self.__cache.cache(self.dataset, self.lumi_mask, baseinfo, result)

        result.stop_on_file_boundary = (result.total_units != total_lumis) and not self.file_based
        if result.stop_on_file_boundary:
            logger.debug("split lumis detected in {} - "
                         "{} unique (run, lumi) but "
                         "{} unique (run, lumi, file) - "
                         "enforcing a limit of one file per task".format(self.dataset, total_lumis, result.total_units))

        return result

    def __init__(self, config):
        BasePlugin.__init__(self, config)

        self.locationDict = {}

        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
                                dbinterface=myThread.dbi)
        self.locationAction = daoFactory(classname="Locations.GetSiteInfo")

        self.packageDir = None

        if os.path.exists(os.path.join(getWMBASE(),
                                       'src/python/WMCore/WMRuntime/Unpacker.py')):
            self.unpacker = os.path.join(getWMBASE(),
                                         'src/python/WMCore/WMRuntime/Unpacker.py')
        else:
            self.unpacker = os.path.join(getWMBASE(),
                                         'WMCore/WMRuntime/Unpacker.py')

        self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
        self.sandbox = None

        self.scriptFile = config.JobSubmitter.submitScript

        self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
        self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
        self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
        self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)

        # Required for global pool accounting
        self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
        self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")

        # Build a requirement string.  All CMS resources match DESIRED_Sites on the START
        # expression side; however, there are currently some resources (T2_CH_CERN_HLT)
        # that are missing the REQUIRED_OS logic.  Hence, we duplicate it here.
        # TODO(bbockelm): Remove reqStr once HLT has upgraded.
        self.reqStr = ('((REQUIRED_OS=?="any") || '
                       '(GLIDEIN_REQUIRED_OS =?= "any") || '
                       'stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS)) && '
                       '(AuthenticatedIdentity =!= "[email protected]")')
        if hasattr(config.BossAir, 'condorRequirementsString'):
            self.reqStr = config.BossAir.condorRequirementsString

        # x509 proxy handling
        proxy = Proxy({'logger': myThread.logger})
        self.x509userproxy = proxy.getProxyFilename()
        self.x509userproxysubject = proxy.getSubject()
        self.x509userproxyfqan = proxy.getAttributeFromProxy(self.x509userproxy)
        # Remove the x509 ads if the job is matching a volunteer resource
        self.x509Expr = 'ifThenElse("$$(GLIDEIN_CMSSite)" =?= "T3_CH_Volunteer",undefined,"%s")'

        return

    def testMyProxyEnvironment(self):
        """
        Test the myProxyEnvironment context manager
        In this test a new Proxy and MyProxy are initialized
        """
        myProxy = Proxy(self.dict)

        # Create the proxy
        myProxy.create()
        proxyPath = myProxy.getProxyFilename()
        userDN = myProxy.getSubject()
        self.assertTrue(os.path.exists(proxyPath))

        # Delegate and check the proxy
        myProxy.delegate(credential=proxyPath, serverRenewer=True)
        valid = myProxy.checkMyProxy()
        self.assertTrue(valid)

        # Make sure X509_USER_PROXY exists only in the context manager and corresponds to a file
        if 'X509_USER_PROXY' in os.environ:
            del os.environ['X509_USER_PROXY']
        self.assertFalse('X509_USER_PROXY' in os.environ)
        with myProxyEnvironment(userDN=userDN, serverCert=serverCert, serverKey=serverKey,
                                myproxySrv='myproxy.cern.ch', proxyDir='/tmp/', logger=self.logger):
            self.assertTrue('X509_USER_PROXY' in os.environ)
            self.assertTrue(os.path.exists(os.environ['X509_USER_PROXY']))
        self.assertFalse('X509_USER_PROXY' in os.environ)

        return

    def validate(self):
        if self.dataset in Dataset.__dsets:
            return True

        if self.lumi_mask:
            self.lumi_mask = self.__get_mask(self.lumi_mask)

        cred = Proxy({'logger': logging.getLogger("WMCore")})
        dbs = DASWrapper(self.dbs_instance, ca_info=cred.getProxyFilename())

        baseinfo = dbs.listFileSummaries(dataset=self.dataset)
        if baseinfo is None or (len(baseinfo) == 1 and baseinfo[0] is None):
            return False
        return True

    def __call__(self):
        server = HTTPRequests(self.serverurl, self.proxyfilename)

        self.logger.debug("Looking up detailed status of task %s" % self.cachedinfo["RequestName"])
        dictresult, status, reason = server.get(self.uri, data={"workflow": self.cachedinfo["RequestName"]})
        dictresult = dictresult["result"][0]  # take just the significant part

        if status != 200:
            msg = "Problem retrieving status:\ninput:%s\noutput:%s\nreason:%s" % (
                str(self.cachedinfo["RequestName"]),
                str(dictresult),
                str(reason),
            )
            raise RESTCommunicationException(msg)

        self.logger.debug(dictresult)  # should be something like {u'result': [[123, u'ciao'], [456, u'ciao']]}

        self.logger.info("Task name:\t\t\t%s" % self.cachedinfo["RequestName"])
        self.logger.info("Task status:\t\t\t%s" % dictresult["status"])

        # Print the url of the panda monitor
        if dictresult["taskFailureMsg"]:
            self.logger.error(
                "%sError during task injection:%s\t%s" % (colors.RED, colors.NORMAL, dictresult["taskFailureMsg"])
            )
        elif dictresult["jobSetID"]:
            p = Proxy({"logger": self.logger})
            username = urllib.quote(p.getUserName())
            self.logger.info(
                "Panda url:\t\t\thttp://panda.cern.ch/server/pandamon/query?job=*&jobsetID=%s&user=%s"
                % (dictresult["jobSetID"], username)
            )

        if dictresult["jobdefErrors"]:
            self.logger.error(
                "%sSubmission partially failed:%s\t%s jobgroup not submittet out of %s:"
                % (colors.RED, colors.NORMAL, dictresult["failedJobdefs"], dictresult["totalJobdefs"])
            )
            for error in dictresult["jobdefErrors"]:
                self.logger.info("\t%s" % error)

        # Print information about jobs
        states = dictresult["jobsPerStatus"]
        total = sum(states[st] for st in states)
        frmt = ""
        for status in states:
            frmt += status + " %s\t" % self._percentageString(states[status], total)
        if frmt:
            self.logger.info("Details:\t\t\t%s" % frmt)

class Proxy(Configurable):

    """
    Wrapper around CMS credentials.

    Parameters
    ----------
        renew : bool
            Whether to renew proxy.  Defaults to `True`.
    """

    _mutable = {}

    def __init__(self, renew=True):
        self.renew = renew
        self.__proxy = WMProxy({'logger': logging.getLogger("WMCore"), 'proxyValidity': '192:00'})
        self.__setup()

    def __setup(self):
        if self.check() and self.__proxy.getTimeLeft() > 4 * 3600:
            if 'X509_USER_PROXY' not in os.environ:
                os.environ['X509_USER_PROXY'] = self.__proxy.getProxyFilename()
        elif self.renew:
            self.__proxy.renew()
            if self.__proxy.getTimeLeft() < 4 * 3600:
                raise AttributeError("could not renew proxy")
            os.environ['X509_USER_PROXY'] = self.__proxy.getProxyFilename()
        else:
            raise AttributeError("please renew or disable your proxy")

    def __getstate__(self):
        state = dict(self.__dict__)
        del state['_Proxy__proxy']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        with PartiallyMutable.unlock():
            self.__proxy = WMProxy({'logger': logging.getLogger("WMCore"), 'proxyValidity': '192:00'})
            self.__setup()

    def check(self):
        left = self.__proxy.getTimeLeft()
        if left == 0:
            return False
        elif left < 4 * 3600:
            logger.warn("only {0}:{1:02} left in proxy lifetime!".format(left / 3600, left / 60))
        return True

    def expires(self):
        return int(time.time()) + self.__proxy.getTimeLeft()

    def time_left(self):
        return self.__proxy.getTimeLeft()

def getProxy(userdn, group, role, defaultDelegation, logger):
    """
    _getProxy_
    """
    log.debug("Retrieving proxy for %s" % userdn)
    proxy = Proxy(defaultDelegation)
    proxyPath = proxy.getProxyFilename( True )
    timeleft = proxy.getTimeLeft( proxyPath )
    if timeleft is not None and timeleft > 3600:
        return (True, proxyPath)
    proxyPath = proxy.logonRenewMyProxy()
    timeleft = proxy.getTimeLeft( proxyPath )
    if timeleft is not None and timeleft > 0:
        return (True, proxyPath)
    return (False, None)

    def createNewMyProxy(self, timeleftthreshold=0, nokey=False):
        """
        Handles the MyProxy creation

        Let the following variables be

        timeleftthreshold: the proxy in myproxy should be delegated for at least this time (14 days)
        myproxytimeleft: current validity of your proxy in myproxy
        usercertDaysLeft: the number of days left before your user certificate expire
        myproxyDesiredValidity: delegate the proxy in myproxy for that time (30 days)

        If we need to renew the proxy in myproxy because its atributes has changed or because it is valid for
        less time than timeleftthreshold then we do it.

        Before doing that, we check when the user certificate is expiring. If it's within the timeleftthreshold (myproxytimeleft < timeleftthreshold)
        we delegate the proxy just for the time we need (checking first if we did not already do it since at some point
        usercertDaysLeft ~= myproxytimeleft and we don't need to delegate it at every command even though myproxytimeleft < timeleftthreshold).

        Note that a warning message is printed at every command it usercertDaysLeft < timeleftthreshold
        """
        myproxy = Proxy ( self.defaultDelegation )
        myproxy.userDN = myproxy.getSubjectFromCert(self.certLocation)

        myproxytimeleft = 0
        self.logger.debug("Getting myproxy life time left for %s" % self.defaultDelegation["myProxySvr"])
        # return an integer that indicates the number of seconds to the expiration of the proxy in myproxy
        myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True, nokey=nokey)
        self.logger.debug("Myproxy is valid: %i" % myproxytimeleft)

        trustRetrListChanged = myproxy.trustedRetrievers!=self.defaultDelegation['serverDN'] #list on the REST and on myproxy are different
        if myproxytimeleft < timeleftthreshold or self.proxyChanged or trustRetrListChanged:
            # checking the enddate of the user certificate
            usercertDaysLeft = myproxy.getUserCertEnddate()
            if usercertDaysLeft == 0:
                msg = "%sYOUR USER CERTIFICATE IS EXPIRED (OR WILL EXPIRE TODAY). CANNOT SUBMIT%s"\
                                        % (colors.RED, colors.NORMAL)
                raise ProxyCreationException(msg)

            #if the certificate is going to expire print a warning. This is going to bre printed at every command if
            #the myproxytimeleft is inferior to the timeleftthreshold
            if usercertDaysLeft < self.myproxyDesiredValidity:
                self.logger.info("%sYour user certificate is going to expire in %s days. Please renew it! %s"\
                                 % (colors.RED, usercertDaysLeft, colors.NORMAL) )
                #check if usercertDaysLeft ~= myproxytimeleft which means we already delegated the proxy for as long as we could
                if abs(usercertDaysLeft*60*60*24 - myproxytimeleft) < 60*60*24 and not trustRetrListChanged: #less than one day between usercertDaysLeft and myproxytimeleft
                    return
                #adjust the myproxy delegation time accordingly to the user cert validity
                self.logger.info("%sDelegating your proxy for %s days instead of %s %s"\
                                 % (colors.RED, usercertDaysLeft, self.myproxyDesiredValidity, colors.NORMAL) )
                myproxy.myproxyValidity = "%i:00" % (usercertDaysLeft*24)

            # creating the proxy
            self.logger.debug("Delegating a myproxy for %s hours" % self.defaultDelegation['myproxyValidity'] )
            try:
                myproxy.delegate(serverRenewer = True, nokey=nokey)
                self.logger.debug("My-proxy delegated.")
            except Exception, ex:
                raise ProxyCreationException("Problems delegating My-proxy. %s"%ex._message)

    def setUp(self):
        """
        Setup for unit tests
        """
        logging.basicConfig(
            level=logging.DEBUG,
            format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
            datefmt="%m-%d %H:%M",
            filename="proxy_unittests.log",
            filemode="w",
        )

        logger_name = "ProxyTest"

        self.logger = logging.getLogger(logger_name)
        self.dict = {
            "logger": self.logger,
            "vo": "cms",
            "group": group,
            "role": role,
            "myProxySvr": myProxySvr,
            "proxyValidity": "192:00",
            "min_time_left": 36000,
            "uisource": uiPath,
        }

        self.proxyPath = None
        self.proxy = Proxy(self.dict)

    def __init__(self, config):
        """
        initialize properties specified from config
        """
        BaseWorkerThread.__init__(self)
        # set the workqueue service for REST call
        self.config = config
        # need to get campaign, user, owner info
        self.agentInfo = initAgentInfo(self.config)
        self.summaryLevel = config.AnalyticsDataCollector.summaryLevel

        proxyArgs = {'logger': logging.getLogger()}
        self.proxy = Proxy(proxyArgs)
        self.proxyFile = self.proxy.getProxyFilename()  # X509_USER_PROXY
        self.userCertFile = self.proxy.getUserCertFilename()  # X509_USER_CERT
        # credential lifetime warning/error thresholds, in days
        self.credThresholds = {'proxy': {'error': 3, 'warning': 5},
                               'certificate': {'error': 10, 'warning': 20}}

        # Monitoring setup
        self.userAMQ = getattr(config.AgentStatusWatcher, "userAMQ", None)
        self.passAMQ = getattr(config.AgentStatusWatcher, "passAMQ", None)
        self.postToAMQ = getattr(config.AgentStatusWatcher, "enableAMQ", False)
        self.topicAMQ = getattr(config.AgentStatusWatcher, "topicAMQ", None)
        self.hostPortAMQ = getattr(config.AgentStatusWatcher, "hostPortAMQ", [('cms-mb.cern.ch', 61313)])

        # T0 doesn't have WorkQueue, so some monitoring/replication code has to be skipped here
        if hasattr(self.config, "Tier0Feeder"):
            self.isT0agent = True
            self.producer = "tier0wmagent"
        else:
            self.isT0agent = False
            self.producer = "wmagent"
            localWQUrl = config.AnalyticsDataCollector.localQueueURL
            self.workqueueDS = WorkQueueDS(localWQUrl)

    def __init__(self, config):
        BasePlugin.__init__(self, config)

        self.locationDict = {}

        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
                                dbinterface=myThread.dbi)
        self.locationAction = daoFactory(classname="Locations.GetSiteInfo")

        self.packageDir = None

        if os.path.exists(os.path.join(getWMBASE(),
                                       'src/python/WMCore/WMRuntime/Unpacker.py')):
            self.unpacker = os.path.join(getWMBASE(),
                                         'src/python/WMCore/WMRuntime/Unpacker.py')
        else:
            self.unpacker = os.path.join(getWMBASE(),
                                         'WMCore/WMRuntime/Unpacker.py')

        self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
        self.sandbox = None

        self.scriptFile = config.JobSubmitter.submitScript

        self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
        self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
        self.jobsPerSubmit = getattr(config.JobSubmitter, 'jobsPerSubmit', 200)
        self.extraMem = getattr(config.JobSubmitter, 'extraMemoryPerCore', 500)

        # Required for global pool accounting
        self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
        self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")

        # Build a requirement string
        self.reqStr = ('stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) '
                       '&& ((REQUIRED_OS=?="any") || stringListMember(GLIDEIN_REQUIRED_OS, REQUIRED_OS))'
                       '&& (TARGET.Cpus >= RequestCpus)')
        if hasattr(config.BossAir, 'condorRequirementsString'):
            self.reqStr = config.BossAir.condorRequirementsString

        # x509 proxy handling
        proxy = Proxy({'logger': myThread.logger})
        self.x509userproxy = proxy.getProxyFilename()
        self.x509userproxysubject = proxy.getSubject()

        return

class Proxy(object):
    '''
    CMS uses proxies constantly. This class is a wrapper function around WMCore
    proxy handling, to allow the user to update/check/delete their proxy in
    myproxy and update/check the local proxy
    '''


    def __init__(self):
        '''
        Constructor
        '''
        self.helper = WMCoreProxy({'logger' : logging})

    def getProxyFilename(self):
        return self.helper.getProxyFilename()
            
    def initProxy(self):
        self.helper.create()
        
    def deleteProxy(self):
        self.helper.destroy()
    
    def uploadToMyproxy(self, allowedDN):
        self.helper.serverDN = allowedDN
        self.helper.delegate( None, True )

def getProxy(config, userdn, group, role):
    """
    _getProxy_
    """
    defaultDelegation = getDefaultDelegation(config, "cms", "myproxy.cern.ch", threading.currentThread().logger)
    defaultDelegation['userDN'] = userdn
    defaultDelegation['group'] = group
    defaultDelegation['role'] = role

    logging.debug("Retrieving proxy for %s" % userdn)
    proxy = Proxy(defaultDelegation)
    proxyPath = proxy.getProxyFilename( True )
    timeleft = proxy.getTimeLeft( proxyPath )
    if timeleft is not None and timeleft > 3600:
        return (True, proxyPath)
    proxyPath = proxy.logonRenewMyProxy()
    timeleft = proxy.getTimeLeft( proxyPath )
    if timeleft is not None and timeleft > 0:
        return (True, proxyPath)
    return (False, None)