Python Dirac.Dirac类代码示例

 def submitProbeJobs(self, ce):
   """ Submit some jobs to the CEs
   """
   
   #need credentials, should be there since the initialize
   
   from DIRAC.Interfaces.API.Dirac import Dirac
   d = Dirac()
   from DIRAC.Interfaces.API.Job import Job
   
   from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
   import DIRAC
   
   ops = Operations()
   scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script)
   
   j = Job()
   j.setDestinationCE(ce)
   j.setCPUTime(1000)
   j.setName("Probe %s" % ce)
   j.setJobGroup("SoftwareProbe")
   j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname), 
                   logFile='SoftwareProbe.log')
   j.setOutputSandbox('*.log')
   res = d.submit(j)
   if not res['OK']:
     return res
     
   return S_OK()

class DiracTestCases(unittest.TestCase):
  """ Dirac API test cases
  """
  def setUp(self):
    self.dirac = Dirac()

  def tearDown(self):
    pass

  def test_basicJob(self):
    jdl = "Parameter=Value;Parameter2=Value2"
    ret = self.dirac._Dirac__getJDLParameters(jdl)
    self.assertTrue(ret['OK'])
    self.assertIn('Parameter', ret['Value'])
    self.assertEqual('Value', ret['Value']['Parameter'])
    self.assertIn('Parameter2', ret['Value'])
    self.assertEqual('Value2', ret['Value']['Parameter2'])

  def test_JobJob(self):
    from DIRAC.Interfaces.API.Job import Job
    job = Job(stdout='printer', stderr='/dev/null')
    ret = self.dirac._Dirac__getJDLParameters(job)
    self.assertTrue(ret['OK'])
    self.assertEqual('printer', ret['Value']['StdOutput'])
    self.assertEqual('/dev/null', ret['Value']['StdError'])

 def do_replicate(self,args):
   """ Replicate a given file to a given SE
       
       usage:
         replicate <LFN> <SE> [<SourceSE>]
   """
   argss = args.split()
   if len(args) < 2:
     print "Error: unsufficient number of arguments"
   lfn = argss[0]
   lfn = self.getPath(lfn)
   se = argss[1]
   sourceSE = ''
   if len(argss)>2:
     sourceSE=argss[2]
   if len(argss)>3: 
     localCache=argss[3]
   try:
     dirac = Dirac()
     result = dirac.replicate(lfn,se,sourceSE,printOutput=True)      
     if not result['OK']:
       print 'Error: %s' %(result['Message'])
     elif not result['Value']:
       print "Replica is already present at the target SE"
     else:  
       print "File %s successfully replicated to the %s SE" % (lfn,se)  
   except Exception, x:
     print "Error: replicate failed with exception: ", x      

 def do_add(self,args):
   """ Upload a new file to a SE and register in the File Catalog
   
       usage:
       
         add <lfn> <pfn> <SE> [<guid>] 
   """
   
   # ToDo - adding directories
   
   argss = args.split()
   
   if len(argss) < 3:
     print "Error: unsufficient number of arguments"
   
   lfn = argss[0]
   lfn = self.getPath(lfn)
   pfn = argss[1]
   se = argss[2]
   guid = None
   if len(argss)>3:
     guid = argss[3]
       
   dirac = Dirac()
   result = dirac.addFile(lfn,pfn,se,guid,printOutput=False)
   if not result['OK']:
     print 'Error: %s' %(result['Message'])
   else:
     print "File %s successfully uploaded to the %s SE" % (lfn,se)  

 def do_get(self,args):
   """ Download file from grid and store in a local directory
   
       usage:
       
         get <lfn> [<local_directory>] 
   """
   
   argss = args.split()
   lfn = argss[0]
   lfn = self.getPath(lfn)
   dir = ''
   if len(argss)>1:
     dir = argss[1]
       
   dirac = Dirac()
   localCWD = ''
   if dir:
     localCWD = os.getcwd()
     os.chdir(dir)
   result = dirac.getFile(lfn)
   if localCWD:
     os.chdir(localCWD)
     
   if not result['OK']:
     print 'Error: %s' %(result['Message'])
   else:
     print "File %s successfully downloaded" % lfn      

def _getOutputs():
  repoLocation = ''
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine( ignoreErrors = False )
  repoLocation = clip.repo
  if not repoLocation:
    Script.showHelp()
    dexit(1)
  from DIRAC import gLogger
  from DIRAC.Interfaces.API.Dirac import Dirac

  dirac = Dirac(True, repoLocation)
  
  exitCode = 0
  res = dirac.monitorRepository(False)
  if not res['OK']:
    gLogger.error("Failed because %s" % res['Message'])
    dexit(1)
    
  res = dirac.retrieveRepositorySandboxes()
  if not res['OK']:
    gLogger.error("Failed because %s" % res['Message'])
    dexit(1)
  if clip.outputdata:
    res = dirac.retrieveRepositoryData()
    if not res['OK']:
      gLogger.error("Failed because %s" % res['Message'])
      exit(1)
  dexit(exitCode)

  def runLocal( self, dirac = None ):
    """ The dirac (API) object is for local submission.
    """

    if dirac is None:
      dirac = Dirac()

    return dirac.submit( self, mode = 'local' )

def __downloadJobDescriptionXML(jobID, basepath):
  """
  Downloads the jobDescription.xml file into the temporary directory
  created.
  
  """
  from DIRAC.Interfaces.API.Dirac import Dirac
  jdXML = Dirac()
  jdXML.getInputSandbox(jobID, basepath)

    def uploadAndRegisterFiles(self, fileList, SE="IHEPD-USER", guid=None, ePoint=""):
        """upload a set of files to SE and register it in DFC.
        user input the directory of localfile.
        argument:
          ePoint is the energy point,for scan data
        we can treat localDir as a kind of datasetName.
        """

        result_OK = 1
        errorList = []
        # fileList = self.getFilenamesByLocaldir(localDir)
        for fullpath in fileList:
            # get the attributes of the file
            fileAttr = self.__getFileAttributes(fullpath)
            if len(fileAttr) == 0:
                print "failed to get file %s attributes" % fullpath
                return S_ERROR("failed to get file attributes")
            # create dir and set dirMetadata to associated dir
            lastDir = self.registerHierarchicalDir(fileAttr, rootDir="/bes")
            dirMeta = self.getDirMetaVal(lastDir)
            if not (dirMeta.has_key("jobOptions") or dirMeta.has_key("description")):
                lastDirMetaDict = {}
                lastDirMetaDict["jobOptions"] = fileAttr["jobOptions"]
                lastDirMetaDict["description"] = fileAttr["description"]
                try:
                    self.__registerDirMetadata(lastDir, lastDirMetaDict)
                except:
                    pass
            if len(ePoint):
                lastDir = lastDir + os.sep + ePoint
            lfn = lastDir + os.sep + fileAttr["LFN"]
            # upload and register file.
            dirac = Dirac()
            result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
            # register file metadata
            if not result["OK"]:
                print "ERROR %s" % (result["Message"])
                # return S_ERROR(result['Message'])
                errorList.append(fullpath)
                result_OK = 0
            else:
                result = self.__registerFileMetadata(lfn, fileAttr)
                if not result["OK"]:
                    result_OK = 0
                    print "failed to register file metadata"
        if result_OK:
            return S_OK()
        else:
            return S_ERROR(errorList)

def _getOutputData():
  cliParams = _Params()
  cliParams.registerSwitches()
  Script.parseCommandLine( ignoreErrors = False )
  if not cliParams.repo:
    Script.showHelp()
    dexit(2)
  from DIRAC.Interfaces.API.Dirac import Dirac
  
  dirac = Dirac(True, cliParams.repo)

  exitCode = 0
  dirac.monitorRepository(False)
  dirac.retrieveRepositoryData()

  dexit(exitCode)

def submitJob(jobPara):
    dirac = Dirac()
    j = Job()
    j.setName(jobPara['jobName'])
    j.setJobGroup(jobPara['jobGroup'])
    j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog'])
    j.setInputSandbox(jobPara['inputSandbox'])
    j.setOutputSandbox(jobPara['outputSandbox'])
    j.setOutputData(jobPara['outputData'], jobPara['SE'])
    j.setDestination(jobPara['sites'])
    j.setCPUTime(jobPara['CPUTime'])
    result = dirac.submit(j)
    if result['OK']:
        print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value'])
    else:
        print 'Job %s submitted failed' %jobPara['jobName']
    return result

    def do_installonsite(self,argss):
        """ Install a release on a grid site : 
            installonsite tag site
        """
        args = argss.split()
        if len(args)<2:
            print self.do_installonsite.__doc__
            return
        tag = args[0]
        site = args[1]
        
        #print "Check if the software with the tag '"+tag+"' exists on the rsync server..." 
        #res = self.client.getSitesForTag(tag)
        #if not res['OK']:
          #print res['Message']
          #return 
        #print "tag found !"  

        from DIRAC.Interfaces.API.Dirac import Dirac
        d = Dirac()
        from DIRAC.Interfaces.API.Job import Job
        
        from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
        import os
        
        ops = Operations()
        scriptname = "InstallSoftware.py"
        j = Job()
        j.setDestination(site)
        j.setCPUTime(1000)
        j.setName("Installation "+tag)
        j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log')
        j.setOutputSandbox('*.log')
        res = d.submit(j)
        if not res['OK']:
          print "Could not submit the installation at site %s, message %s"%(site,res['Message'])
          return
        
        print "Job submitted, id = "+str(res['Value'])
        
        print "Add tag :"
        res = self.client.addTagAtSite(tag,site)
        if not res['OK']:
            print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message'])
            return
        print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))

def main():
  """ Main program entry point. """
  if len(sys.argv) < 2 or len(sys.argv) > 3:
    usage()
  uname = sys.argv[1]
  site = None
  if len(sys.argv) >= 3:
    site = sys.argv[2]
  print "Fetching job list for user '%s'..." % uname
  jlist = find_jobs(uname, site)
  jlist.append('1')
  print "Found %u jobs, killing..." % len(jlist)
  dirac = Dirac()
  for loc in xrange(0, len(jlist), BATCH_SIZE):
    print "%u/%u complete." % (loc, len(jlist))
    dirac.killJob(jlist[loc:loc+BATCH_SIZE])
  print "%u/%u complete." % (len(jlist), len(jlist))
  print "Exiting."

  def submitNewBigJob( self ):

    result = jobDB.getJobJDL( str( self.__jobID ) , True )
    classAdJob = ClassAd( result['Value'] )
    executableFile = ""
    if classAdJob.lookupAttribute( 'Executable' ):
      executableFile = classAdJob.getAttributeString( 'Executable' )

    tempPath = self.__tmpSandBoxDir
    dirac = Dirac()
    if not os.path.exists( tempPath ):
      os.makedirs( tempPath )

    settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
    self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
    moveData = self.__tmpSandBoxDir + "/InputSandbox" + str( self.__jobID )

    HiveV1Cli = HiveV1Client( self.__User , self.__publicIP )
    returned = HiveV1Cli.dataCopy( moveData, self.__tmpSandBoxDir )
    self.log.info( 'Copy the job contain to the Hadoop Master with HIVE: ', returned )

    jobInfo = jobDB.getJobAttributes( self.__jobID )
    if not jobInfo['OK']:
      return S_ERROR( jobInfo['Value'] )
    proxy = ""
    jobInfo = jobInfo['Value']
    if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
      proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
    else:
      proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )

    HiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile
    HiveJobOutput = str( self.__jobID ) + "_" + executableFile + "_out"

    returned = HiveV1Cli.jobSubmit( tempPath, HiveJob, proxy['chain'], HiveJobOutput )
    self.log.info( 'Launch Hadoop-Hive job to the Master: ', returned )

    if not returned['OK']:
      return S_ERROR( returned['Message'] )
    else:
      self.log.info( 'Hadoop-Hive Job ID: ', returned['Value'] )

    return S_OK( returned['Value'] )

    def downloadFilesByFilelist(self, fileList, destDir=""):
        """downLoad a set of files form SE.
        use getFilesByFilelist() get a list of lfns and download these files.
        fileList get from function getFilesByDatesetName()

           Example usage:
           >>>badger.downloadFilesByFilelist(fileList)
        """
        errorDict = {}
        dirac = Dirac()
        # fileList = self.getFilesByDatasetName(dataset_name)
        for lfn in fileList:
            result = dirac.getFile(lfn, destDir, printOutput=False)
            if not result["OK"]:
                errorDict[lfn] = result["Message"]
        if errorDict:
            serr = S_ERROR()
            serr["errorDict"] = errorDict
            return serr
        else:
            return S_OK("File download successfully.")