Python DataManager.putAndRegister方法代码示例

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
    def finish(self):
        """
        after having set all the files, this one does all the job
        @return:
        """
        rc = 0
        rm = DataManager()
        for item in self.listFileStaged:
            # print("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'")
            if not self.SE:
                self.log.info("No SE available for '" + item[0] + "'")
                rc += 1
                continue
            else:
                self.log.info("Trying to store '" + item[0] + "' in SE : '" + self.SE + "' ...")
                result = rm.putAndRegister(item[1], item[0], self.SE)
                if not result["OK"]:
                    self.log.info("ERROR %s" % (result["Message"]))

                    self.log.info("Wait 5sec before trying again...")
                    time.sleep(5)
                    result = rm.putAndRegister(item[1], item[0], self.SE)
                    if not result["OK"]:
                        self.log.info("ERROR %s" % (result["Message"]))
                        while not result["OK"]:
                            self.listSEs.remove(self.SE)  # make sure not to pick the same SE again.
                            self.__pickRandomSE()
                            if not self.SE:
                                rc += 1
                                break
                            self.log.info("Trying with another SE : '" + self.SE + "' . In 5sec...")
                            time.sleep(5)
                            result = rm.putAndRegister(item[1], item[0], self.SE)
                            if result["OK"]:
                                self.log.info("file stored : '" + item[1] + "' in '" + self.SE + "'")
                            else:
                                self.log.error("ERROR : failed to store the file '" + item[1] + "' ...")
                                rc += 1

        return rc

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
    def _add_file(self, lfn, localfile, SE, guid=None):
        dm = DataManager()
        self._create_test_file()
        if not os.path.exists(self.options['test_file']):
            gLogger.error("File %s must exist locally" % localfile)
        if not os.path.isfile(self.options['test_file']):
            gLogger.error("%s is not a file" % localfile)

        res = dm.putAndRegister(lfn, localfile, SE, guid)
        if not res['OK']:
            gLogger.error('Error: failed to upload %s to %s' % (lfn, SE))
            return S_ERROR(res['Message'])
        return S_OK(res['Value']['Successful'][lfn])

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
class ReplicaManagerTestCase(unittest.TestCase):
  """ Base class for the Replica Manager test cases
  """
  def setUp(self):
    self.dataManager = DataManager()
    self.fileName = '/tmp/temporaryLocalFile'
    file = open(self.fileName,'w')
    file.write("%s" % time.time())
    file.close()

  def test_putAndRegister(self):
    print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time()
    diracSE = 'GRIDKA-RAW'
    putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
    removeRes = self.dataManager.removeFile(lfn)

    # Check that the put was successful
    self.assert_(putRes['OK'])
    self.assert_(putRes['Value'].has_key('Successful'))
    self.assert_(putRes['Value']['Successful'].has_key(lfn))
    self.assert_(putRes['Value']['Successful'][lfn])
    # Check that the removal was successful
    self.assert_(removeRes['OK'])
    self.assert_(removeRes['Value'].has_key('Successful'))
    self.assert_(removeRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeRes['Value']['Successful'][lfn])

  def test_putAndRegisterReplicate(self):
    print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time()
    diracSE = 'GRIDKA-RAW'
    putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
    replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='')
    removeRes = self.dataManager.removeFile(lfn)

    # Check that the put was successful
    self.assert_(putRes['OK'])
    self.assert_(putRes['Value'].has_key('Successful'))
    self.assert_(putRes['Value']['Successful'].has_key(lfn))
    self.assert_(putRes['Value']['Successful'][lfn])
    # Check that the replicate was successful
    self.assert_(replicateRes['OK'])
    self.assert_(replicateRes['Value'].has_key('Successful'))
    self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
    self.assert_(replicateRes['Value']['Successful'][lfn])
    # Check that the removal was successful
    self.assert_(removeRes['OK'])
    self.assert_(removeRes['Value'].has_key('Successful'))
    self.assert_(removeRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeRes['Value']['Successful'][lfn])

  def test_putAndRegisterGetReplicaMetadata(self):
    print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time()
    diracSE = 'GRIDKA-RAW'
    putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
    metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE)
    removeRes = self.dataManager.removeFile(lfn)

    # Check that the put was successful
    self.assert_(putRes['OK'])
    self.assert_(putRes['Value'].has_key('Successful'))
    self.assert_(putRes['Value']['Successful'].has_key(lfn))
    self.assert_(putRes['Value']['Successful'][lfn])
    # Check that the metadata query was successful
    self.assert_(metadataRes['OK'])
    self.assert_(metadataRes['Value'].has_key('Successful'))
    self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
    self.assert_(metadataRes['Value']['Successful'][lfn])
    metadataDict = metadataRes['Value']['Successful'][lfn]
    self.assert_(metadataDict.has_key('Cached'))
    self.assert_(metadataDict.has_key('Migrated'))
    self.assert_(metadataDict.has_key('Size'))
    # Check that the removal was successful
    self.assert_(removeRes['OK'])
    self.assert_(removeRes['Value'].has_key('Successful'))
    self.assert_(removeRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeRes['Value']['Successful'][lfn])


  def test_putAndRegsiterGetAccessUrl(self):
    print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time()
    diracSE = 'GRIDKA-RAW'
    putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
    getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE)
    print getAccessUrlRes
    removeRes = self.dataManager.removeFile(lfn)

    # Check that the put was successful
    self.assert_(putRes['OK'])
    self.assert_(putRes['Value'].has_key('Successful'))
    self.assert_(putRes['Value']['Successful'].has_key(lfn))
    self.assert_(putRes['Value']['Successful'][lfn])
    # Check that the access url was successful
    self.assert_(getAccessUrlRes['OK'])
    self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
    self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
    self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
class SETest( TestBase ):
  """
    SETest is used to test the availability of SE.
  """
    
  def __init__( self, args = None, apis = None ):
    super( SETest, self ).__init__( args, apis )
    
    self.__lfnPath = '/bes/user/z/zhaoxh/'
    self.__testFile = 'test.dat'
    self.__localPath = '/tmp/'
    
    if 'DataManager' in self.apis:
      self.dm = self.apis[ 'DataManager' ]
    else:
      self.dm = DataManager()
    
    
  def doTest( self, elementDict ):
    """
      Test upload and download for specified SE.
    """
    
    elementName = elementDict[ 'ElementName' ]

    testFilePath = self.__localPath + self.__testFile
    if not os.path.exists( testFilePath ) or not os.path.isfile( testFilePath ):
      f = open( testFilePath, 'w' )
      f.write( 'hello' )
      f.close()
        
    status = 'OK'
    log = ''
    lfnPath = self.__lfnPath + elementName + '-' + self.__testFile
    submissionTime = datetime.utcnow().replace( microsecond = 0 )
    
    LOCK.acquire()
    start = time.time()
    result = self.dm.putAndRegister( lfnPath, testFilePath, elementName )
    uploadTime = time.time() - start
    if result[ 'OK' ]:
      log += 'Succeed to upload file to SE %s.\n' % elementName
      log += 'Upload Time : %ss\n' % uploadTime
      
      start = time.time()
      result = self.dm.getReplica( lfnPath, elementName, self.__localPath )
      downloadTime = time.time() - start
      if result[ 'OK' ]:
        log += 'Succeed to download file from SE %s.\n' % elementName
        log += 'Download Time : %ss\n' % downloadTime
      else:
        status = 'Bad'
        log += 'Failed to download file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
      
      result = self.dm.removeFile( lfnPath )
      if result[ 'OK' ]:
        log += 'Succeed to delete file from SE %s.\n' % elementName
      else:
        log += 'Faile to delete file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
        
    else:
      status = 'Bad'
      log += 'Failed to upload file to SE %s : %s\n' % ( elementName, result[ 'Message' ] )
    LOCK.release()
      
    completionTime = datetime.utcnow().replace( microsecond = 0 )
    applicationTime = ( completionTime - submissionTime ).total_seconds()
    
    result = { 'Result' : { 'Status' : status,
                           'Log' : log,
                           'SubmissionTime' : submissionTime,
                           'CompletionTime' : completionTime,
                           'ApplicationTime' : applicationTime },
              'Finish' : True }

    localFile = self.__localPath + elementName +'-' + self.__testFile
    if os.path.exists( localFile ) and os.path.isfile( localFile ):
      os.remove( localFile )
      
    return S_OK( result )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]

#.........这里部分代码省略.........
  if not clip.storageElement:
    gLogger.error('You need a storage element')
    Script.showHelp()
    dexit(1)
  
  for key in MANDATORY_KEYS:
    if key not in clip.fmeta:
      gLogger.error("Not all mandatory meta data defined, please check and add key: ", key)
      Script.showHelp()
      dexit(1)
    
  #resolve the inout files
  flist = []
  if os.path.isdir(clip.dir):
    flistd = os.listdir(clip.dir)
    for filename in flistd:
      if filename.count(".stdhep"):
        flist.append( os.path.join(clip.dir, filename) )
  elif os.path.isfile(clip.dir):
    flist.append(clip.dir)
  else:
    gLogger.error("%s is not a file nor a directory" % clip.dir)
    dexit(1)  
  
  gLogger.notice("Will eventually upload %s file(s)" % len(flist))
    
  from DIRAC.Core.Utilities.PromptUser import promptUser
    
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
  basepath = Operations().getValue('Production/ILC_ILD/BasePath','')
  if not basepath:
    gLogger.error('Failed to contact CS, please try again')
    dexit(1)
  
  basepath = "/".join(basepath.split("/")[:-2])+"/" #need to get rid of the ild/ part at the end
    
  finalpath = os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass, str(clip.fmeta['GenProcessID']))
  gLogger.notice("Will upload the file(s) under %s" % finalpath)
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)
  
  dirmeta = []
  dirmeta.append({'path':os.path.join(basepath, 'generated'), 'meta':{'Datatype':'gen'}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams), 'meta':{'Energy':clip.energy, 'MachineParams':clip.machineParams}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass), 'meta':{'EvtClass':clip.evtclass }})
  dirmeta.append({'path':finalpath, 'meta': {'EvtType':clip.evttype ,'Luminosity':clip.lumi, 'ProcessID': clip.fmeta['GenProcessID']} })
  
  final_fname_base = 'E'+clip.energy+"-"+clip.machineParams+".P"+clip.fmeta['GenProcessName']+".G"+clip.fmeta['ProgramNameVersion'] + "."+clip.particle1+clip.pol1+"."+clip.particle2+clip.pol2+".I"+str(clip.fmeta['GenProcessID'])
  gLogger.notice("Final file name(s) will be %s where XX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XX.ext") )
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)    

  
  from DIRAC.DataManagementSystem.Client.DataManager import DataManager
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  
  for pathdict in dirmeta:
    res = fc.createDirectory(pathdict['path'])
    if not res['OK']:
      gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] )
      dexit(0)

    res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    if not res['OK']:
      gLogger.error( "Failed to set meta data %s to %s\n" %(pathdict['meta'], pathdict['path']), res['Message'] )

  datMan = DataManager()
  for filename in flist:
    fnum = filename.split(".")[-2]
    fext = filename.split(".")[-1]
    final_fname = final_fname_base + '.' + fnum + "." + fext
    gLogger.notice("Uploading %s to" % filename, finalpath+"/"+final_fname)
    if not clip.force:
      res = promptUser('Continue?', ['y','n'], 'n')
      if not res['OK']:
        gLogger.error(res['Message'])
        break
      if not res['Value'].lower()=='y':
        break    

    res = datMan.putAndRegister(finalpath+"/"+final_fname, filename, clip.storageElement)
    if not res['OK']:
      gLogger.error("Failed to upload %s:" % filename, res['Message'])
      continue
    res = fc.setMetadata(finalpath+"/"+final_fname, clip.fmeta)
    if not res['OK']:
      gLogger.error("Failed setting the metadata to %s:" % filename, res['Message'])
      
  dexit(0)

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
    inputFile.close()
else:
  lfns.append( getDict( args ) )

from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0

dm = DataManager()
for lfn in lfns:
  if not os.path.exists( lfn['localfile'] ):
    gLogger.error( "File %s must exist locally" % lfn['localfile'] )
    exitCode = 1
    continue
  if not os.path.isfile( lfn['localfile'] ):
    gLogger.error( "%s is not a file" % lfn['localfile'] )
    exitCode = 2
    continue

  gLogger.notice( "\nUploading %s" % lfn['lfn'] )
  res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'] )
  if not res['OK']:
    exitCode = 3
    gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) )
    continue
  else:
    gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] )

DIRAC.exit( exitCode )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
    inputFile.close()
else:
  lfns.append( getDict( args ) )

from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0

dm = DataManager()
for lfn in lfns:
  if not os.path.exists( lfn['localfile'] ):
    gLogger.error( "File %s must exist locally" % lfn['localfile'] )
    exitCode = 1
    continue
  if not os.path.isfile( lfn['localfile'] ):
    gLogger.error( "%s is not a file" % lfn['localfile'] )
    exitCode = 2
    continue

  gLogger.notice( "\nUploading %s" % lfn['lfn'] )
  res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite = overwrite )
  if not res['OK']:
    exitCode = 3
    gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) )
    continue
  else:
    gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] )

DIRAC.exit( exitCode )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import putAndRegister [as 别名]
from DIRAC import S_OK, S_ERROR, gLogger, exit

from DIRAC.DataManagementSystem.Client.DataManager      import DataManager

lfn = args[0]
pfn = args[1]
se = args[2]

exit_code = 0
log = ''

dm = DataManager()

start = time.time()
result = dm.removeFile( lfn )
result = dm.putAndRegister( lfn, pfn, se )
uploadTime = time.time() - start
if result[ 'OK' ]:
  log += 'Succeed to upload file to SE %s.\n' % se
  log += 'Upload Time : %ss\n' % uploadTime

  start = time.time()
  result = dm.getReplica( lfn, se, tempfile.gettempdir() )
  downloadTime = time.time() - start
  if result[ 'OK' ]:
    log += 'Succeed to download file from SE %s.\n' % se
    log += 'Download Time : %ss\n' % downloadTime
  else:
    exit_code = 1
    log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )