Python TimerCommand.TimerCommand类代码示例

    def _stagefile(self, cmd, source, destination, filesize, is_stagein):
        """
            Stage the file
            mode is stagein or stageout
            :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
            :raise: PilotException in case of controlled error
        """

        timeout = self.getTimeOut(filesize)

        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        self.log("Executing command: %s, timeout=%s" % (cmd, timeout))

        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: %s threw an exception: %s" % (cmd, e))
            rcode, output = -1, str(e)

    def _stagefile(self, source, destination, filesize, is_stagein):
        """
            Stage the file
            mode is stagein or stageout
            :return: destination file details (checksum, checksum_type) in case of success, throw exception in case of failure
            :raise: PilotException in case of controlled error
        """

        if self.checksum_type not in ["adler32"]:  # exclude md5
            raise PilotException(
                "Failed to stage file: internal error: unsupported checksum_type=%s .. " % self.checksum_type,
                code=PilotErrors.ERR_STAGEINFAILED if is_stagein else PilotErrors.ERR_STAGEOUTFAILED,
                state="BAD_CSUMTYPE",
            )

        cmd = "%s -np -f %s %s %s" % (self.copy_command, self.coption, source, destination)
        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        timeout = self.getTimeOut(filesize)
        self.log("Executing command: %s, timeout=%s" % (cmd, timeout))

        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: xrdcp threw an exception: %s" % e)
            rcode, output = -1, str(e)

    def removeRemoteFile(self, surl):
        """
            Do remove (remote) file from storage
            :raise: PilotException in case of controlled error
        """

        # take a 1 m nap before trying to reach the file (it might not be available immediately after a transfer)
        self.log("INFO: [gfal removeRemoteFile] Taking a 1 m nap before the file removal attempt")
        time.sleep(60)

        timeout = self.getTimeOut(0)
        cmd = 'gfal-rm --verbose -t %s %s' % (timeout, surl)

        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        self.log("Do remove RemoteFile: %s" % surl)
        self.log("Executing command: %s, timeout=%s" % (cmd, timeout))

        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
            rcode, output = -1, str(e)

    def stageInFile(self, source, destination, sourceSize, sourceChecksum, guid=None):
        """StageIn the file. should be implementated by different site mover."""
        statusRet = 0
        outputRet = {}
        outputRet["errorLog"] = None
        outputRet["report"] = {}
        outputRet["report"]["clientState"] = None

        # build the parameters
        _params = ""
        if sourceSize != 0 and sourceSize != "0":
            _params += self.__par_filesize % (sourceSize)
        if sourceChecksum and sourceChecksum != 'None' and sourceChecksum != 0 and sourceChecksum != "0" and not self.isDummyChecksum(sourceChecksum):
            csumtype = self.getChecksumType(sourceChecksum)
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                csumtype = 'md5'
            _params += self.__par_checksum % ("%s:%s" % (csumtype, sourceChecksum),)
        # add the guid option
        _params += " --guid %s" % (guid)

        self.log("StageIn files started.")
        _cmd_str = self.__localget % (self._setup, _params, source, destination)
        self.log('Executing command: %s' % (_cmd_str))
        s = -1
        o = '(not defined)'
        t0 = os.times()
        outputRet["report"]['relativeStart'] = time()
        outputRet["report"]['transferStart'] = time()
        try:
            timerCommand = TimerCommand(_cmd_str)
            s, o = timerCommand.run(timeout=self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
            o = str(e)

    def stageInFile(self, source, destination):
        """StageIn the file. should be implementated by different site mover."""
        statusRet = 0
        outputRet = {}
        outputRet["errorLog"] = None
        outputRet["report"] = {}
        outputRet["report"]["clientState"] = None

        self.log("StageIn files started.")
        _cmd_str = '%s xrdcp -np %s %s' % (self._setup, source, destination)

        # update job setup script
        thisExperiment = getExperiment(self.__experiment)
        # add the full stage-out command to the job setup script
        to_script = _cmd_str.replace(destination, "`pwd`/%s" % os.path.basename(destination))
        to_script = to_script.lstrip(' ') # remove any initial spaces
        if to_script.startswith('/'):
            to_script = 'source ' + to_script
        thisExperiment.updateJobSetupScript(os.path.dirname(destination), to_script=to_script)

        self.log('Executing command: %s' % (_cmd_str))
        s = -1
        o = '(not defined)'
        t0 = os.times()
        outputRet["report"]['relativeStart'] = time()
        outputRet["report"]['transferStart'] = time()
        try:
            timerCommand = TimerCommand(_cmd_str)
            s, o = timerCommand.run(timeout=self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
            o = str(e)

 def stageOutFile(self, source, destination, sourceSize=None, sourceChecksum=None, token=None, timeout=3600):
     if self._useTimerCommand:
         timerCommand = TimerCommand()
         ret = timerCommand.runFunction(self.s3StageOutFile, args=(source, destination, sourceSize, sourceChecksum, token), timeout=timeout)
         return ret
     else:
         return self.s3StageOutFile(source, destination, sourceSize, sourceChecksum, token)

    def stageOutFile(self, source, destination, sourceSize, sourceChecksum, checksumType, guid, token=None):
        """Stage out the file. Should be implementated by different site mover"""
        statusRet = 0
        outputRet = {}
        outputRet["errorLog"] = None
        outputRet["report"] = {}
        outputRet["report"]["clientState"] = None

        # build the parameters
        _params = ""
        if token:
            # Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
            if "dst:" in token:
                token = token[len('dst:'):]
                tolog("Dropped dst: part of space token descriptor; token=%s" % (token))
                if 'DATADISK' in token:
                    token = "ATLASDATADISK"
                else:
                    token = "ATLASGROUPDISK"
                tolog("Space token descriptor reset to: %s" % (token))

            _params = self.__spacetoken % (token)
        if sourceSize != 0 and sourceSize != "0":
            _params += self.__par_filesize % (sourceSize)
        if sourceChecksum:
            _params += self.__par_checksum % ("%s:%s" % (checksumType, sourceChecksum),)
        # add the guid option
        _params += " --guid %s" % (guid)

        if ".log." in destination:
            _cmd_str = self.__localput % (self._setup, _params, source, destination)
        else:
            _cmd_str = self.__localputBAD % (self._setup, _params, source, destination)

        tolog("Executing command: %s" % (_cmd_str))
        ec = -1
        t0 = os.times()
        o = '(not defined)'
        outputRet["report"]['relativeStart'] = time()
        outputRet["report"]['transferStart'] =  time()
        try:
            timerCommand = TimerCommand(_cmd_str)
            ec, o = timerCommand.run(timeout=self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! lsm command threw an exception: %s" % (o))
            o = str(e)

    def stageOutFile(self, source, destination, token=None):
        """Stage out the file. Should be implementated by different site mover"""
        statusRet = 0
        outputRet = {}
        outputRet["errorLog"] = None
        outputRet["report"] = {}
        outputRet["report"]["clientState"] = None
        outputRet["output"] = None


        command = "%s xrdcp -h" % (self._setup)
        tolog("Execute command(%s) to decide whether -adler or --cksum adler32 to be used." % command)
        status_local, output_local = commands.getstatusoutput(command)
        tolog("status: %s, output: %s" % (status_local, output_local))
        checksum_option = ""
        if "-adler" in output_local:
            checksum_option = " -adler "
        elif "--cksum" in output_local:
            checksum_option = " --cksum adler32 "
        #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites
        if checksum_option != "":
            tolog("Use (%s) to get the checksum" % checksum_option)
        else:
            tolog("Cannot find -adler nor --cksum. will not use checksum")
        #checksum_option = " -adler " # currently use this one. --cksum will fail on some sites

        # surl is the same as putfile
        _cmd_str = '%s xrdcp -np -f %s %s %s' % (self._setup, checksum_option, source, destination)


        tolog("Executing command: %s" % (_cmd_str))
        ec = -1
        t0 = os.times()
        o = '(not defined)'
        outputRet["report"]['relativeStart'] = time()
        outputRet["report"]['transferStart'] =  time()
        try:
            timerCommand = TimerCommand(_cmd_str)
            ec, o = timerCommand.run(timeout=self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! xrdcp threw an exception: %s" % (o))
            o = str(e)

    def isFileStaged(self, fspec):

        is_staged = True # assume file is staged by default

        cmd = '%s -P -t -1 %s' % (self.copy_command, fspec.turl)
        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        timeout = 10
        self.log("Executing command: %s, timeout=%s" % (cmd, timeout))

        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: %s threw an exception: %s" % (self.copy_command, e))
            rcode, output = -1, str(e)

    def stageIn(self, source, destination, fspec):
        """
        Query HTTP for etag, then symlink to the pilot working directory.

        :param source:      original file location
        :param destination: where to create the link
        :param fspec:       dictionary containing destination replicas, scope, lfn
        :return:            destination file details (checksumtype, checksum, size)
        """

        self.log('source: %s' % str(source))
        self.log('destination: %s' % str(destination))
        self.log('fspec: %s' % str(fspec))
        self.log('fspec.scope: %s' % str(fspec.scope))
        self.log('fspec.lfn: %s' % str(fspec.lfn))
        self.log('fspec.ddmendpoint: %s' % str(fspec.ddmendpoint))

        # figure out the HTTP SURL from Rucio

        from rucio.client import ReplicaClient

        rc = ReplicaClient()
        http_surl_reps = [r for r in rc.list_replicas(dids=[{'scope': fspec.scope,
                                                             'name': fspec.lfn}],
                                                      schemes=['davs'],
                                                      rse_expression=fspec.ddmendpoint)]
        self.log('http_surl_reps: %s' % http_surl_reps)

        http_surl = http_surl_reps[0]['rses'][fspec.ddmendpoint][0].rsplit('_-')[0]
        self.log('http_surl: %s' % http_surl)

        # retrieve the TURL from the webdav etag
        cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % http_surl
        self.log('ETAG retrieval: %s' % cmd)
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=10)
        except Exception, e:
            self.log('FATAL: could not retrieve STORM WebDAV ETag: %s' % e)
            raise PilotException('Could not retrieve STORM WebDAV ETag: %s' % e)

    def stageInFile(self, source, destination):
        """StageIn the file. should be implementated by different site mover."""
        statusRet = 0
        outputRet = {}
        outputRet["errorLog"] = None
        outputRet["report"] = {}
        outputRet["report"]["clientState"] = None

        self.log("StageIn files started.")
        _cmd_str = '%s xrdcp %s %s' % (self._setup, source, destination)
        self.log('Executing command: %s, timeout: %s' % (_cmd_str, self.timeout))
        s = -1
        o = '(not defined)'
        t0 = os.times()
        outputRet["report"]['relativeStart'] = time()
        outputRet["report"]['transferStart'] = time()
        try:
            timerCommand = TimerCommand(_cmd_str)
            s, o = timerCommand.run(timeout=self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
            o = str(e)

    def getRemoteFileSize(self, filename):
        """
            get size of remote file
            Should be implemented by different site mover
            :return: length of file
            :raise: an exception in case of errors
        """

        # do not rely on gfal-ls internal time out
        timeout = self.getTimeOut(0)
        cmd = "gfal-ls -l -t %d %s" % (timeout, filename)
        self.log("getRemoteFileSize: execute command: %s" % cmd )

        timer = TimerCommand(cmd)
        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: %s threw an exception: %s" % ('gfal-rm', e))
            rcode, output = -1, str(e)

    def stageOutFile(self, source, destination):
        """
            Stage out the file
            Should be implementated by different site mover
            :return: remote file (checksum, checksum_type) in case of success, throw exception in case of failure
            :raise: PilotException in case of controlled error
        """

        if self.checksum_type not in ["adler32"]:  # exclude md5
            raise PilotException(
                "Failed to stageOutFile(): internal error: unsupported checksum_type=%s .. " % self.checksum_type,
                code=PilotErrors.ERR_STAGEOUTFAILED,
                state="BAD_CSUMTYPE",
            )

        cmd = "%s -h" % self.copy_command
        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        self.log("Execute command (%s) to decide which option should be used to calc file checksum.." % cmd)

        c = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
        output = c.communicate()[0]

        self.log("status: %s, output: %s" % (c.returncode, output))

        coption = ""

        if c.returncode:
            self.log("FAILED to execute command=%s: %s" % (cmd, output))
        else:
            if "--cksum" in output:
                coption = "--cksum %s:print" % self.checksum_type
            elif "-adler" in output and self.checksum_type == "adler32":
                coption = "-adler"
            elif "-md5" in output and self.checksum_type == "md5":
                coption = "-md5"

        if coption:
            self.log("Use %s option to get the checksum" % coption)
        else:
            self.log("Cannot find neither -adler nor --cksum. will not use checksum")

        cmd = "%s -np -f %s %s %s" % (self.copy_command, coption, source, destination)
        setup = self.getSetup()
        if setup:
            cmd = "%s; %s" % (setup, cmd)

        timeout = self.getTimeOut(os.path.getsize(source))
        self.log("Executing command: %s, timeout=%s" % (cmd, timeout))

        t0 = datetime.now()
        is_timeout = False
        try:
            timer = TimerCommand(cmd)
            rcode, output = timer.run(timeout=timeout)
            is_timeout = timer.is_timeout
        except Exception, e:
            self.log("WARNING: xrdcp threw an exception: %s" % e)
            rcode, output = -1, str(e)

 def stageInFile(self, source, destination, sourceSize=None, sourceChecksum=None):
     timerCommand = TimerCommand()
     ret = timerCommand.runFunction(self.s3StageInFile, args=(source, destination, sourceSize, sourceChecksum), timeout=600)
     return ret