Python saga.url函数代码示例

    def file_stage_in_with_saga(self, input_file_list_with_path, remote_machine_ip, remote_dir):
        cwd = os.getcwd()
        for ifile in input_file_list_with_path:
            # destination url
            if remote_machine_ip.find("localhost") >= 0:
                dest_url_str = "file://"
            else:
                dest_url_str = "gridftp://" + remote_machine_ip + "/"
            ifile_basename = os.path.basename(ifile)
            try:
                dest_dir = dest_url_str + remote_dir
                saga.file.directory(saga.url(dest_dir), saga.file.Create | saga.file.ReadWrite)
            except:
                print "Could not create: " + dest_dir

            dest_url_str = dest_url_str + os.path.join(remote_dir, ifile_basename)
            # source url
            source_url_str = "file://" + os.path.join(cwd, ifile)

            if not os.path.isfile(ifile):
                error_msg = "Input file %s does not exist in %s" % (ifile_basename, os.path.dirname(ifile))
                logging.error(error_msg)
            else:
                try:
                    source_url = saga.url(source_url_str)
                    dest_url = saga.url(dest_url_str)
                    print "stage file: " + source_url_str + " to " + dest_url_str
                    sagafile = saga.file.file(source_url)
                    sagafile.copy(dest_url)
                    logging.info("Now Input file %s is staged into %s" % (ifile_basename, dest_url_str))
                except saga.exception, e:
                    error_msg = "Input file %s failed to be staged in" % (ifile_basename)
                    logging.error(error_msg)

    def start_pilot_job(self, 
                 lrms_url, 
                 bigjob_agent_executable,
                 number_nodes,
                 queue,
                 project,
                 working_directory,
                 userproxy,
                 walltime):
        """ start advert_launcher on specified host """
        if userproxy != None and userproxy != '':
            os.environ["X509_USER_PROXY"]=userproxy
            print "use proxy: " + userproxy
        else:
            print "use standard proxy"

        #register advert entry
        lrms_saga_url = saga.url(lrms_url)
        self.pilot_url = self.app_url.get_string() + "/" + lrms_saga_url.host
        print "create advert entry: " + self.pilot_url
        self.pilot_dir = saga.advert.directory(saga.url(self.pilot_url), saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
        # application level state since globus adaptor does not support state detail
        self.pilot_dir.set_attribute("state", str(saga.job.Unknown)) 
        logging.debug("set pilot state to: " + self.pilot_dir.get_attribute("state"))
        self.number_nodes=int(number_nodes)        
 
        # create job description
        jd = saga.job.description()
        jd.number_of_processes = str(number_nodes)
        jd.spmd_variation = "single"
        jd.arguments = [bigjob_agent_executable, self.database_host, self.pilot_url]
        jd.executable = "/bin/bash"
        #jd.executable = bigjob_agent_executable
        if queue != None:
            jd.queue = queue
        if project !=None:
            jd.job_project = [project]
        if walltime!=None:
            jd.wall_time_limit=str(walltime)

        if working_directory != None:
            jd.working_directory = working_directory
        else:
            jd.working_directory = "$(HOME)"
            
            
      #  if not os.path.isdir(jd.working_directory):
       #     os.mkdir(jd.working_directory)
            
        print "Working directory: " + jd.working_directory
        
        jd.output = "stdout-bigjob_agent-" + str(self.uuid) + ".txt"
        jd.error = "stderr-bigjob_agent-" + str(self.uuid) + ".txt"
           
        # Submit jbo
        js = saga.job.service(lrms_saga_url)
        self.job = js.create_job(jd)
        print "Submit pilot job to: " + str(lrms_saga_url)
        self.job.run()
        return self.job

    def file_stage_in_with_saga(self, input_file_list_with_path, remote_url_prefix, remote_dir):
        cwd = os.getcwd()
        for ifile in input_file_list_with_path:
            # destination url
            dest_url = saga.url(remote_url_prefix + "/")
            ifile_basename = os.path.basename(ifile)

            try:
                dest_dir = saga.url(remote_url_prefix)
                dest_dir.path = remote_dir
                saga.file.directory(dest_dir, saga.file.Create |  saga.file.ReadWrite)
            except:
                print "Could not create: " + dest_dir.get_string()

            dest_url.path = os.path.join(remote_dir, ifile_basename)

            # source url
            source_url = saga.url('file://' + os.path.join(cwd, ifile))

            if not os.path.isfile(ifile):
                error_msg = "Input file %s does not exist in %s"%(ifile_basename, os.path.dirname(ifile))
                print(error_msg)
            else:
                try:
                    print "stage file: " + source_url.get_string() + " to " + dest_url.get_string()
                    sagafile = saga.filesystem.file(source_url)
                    sagafile.copy(dest_url)
                except saga.exception, e:
                    error_msg = "Input file %s failed to be staged in"%(ifile_basename)
                    print(error_msg)

 def dequeue_job(self, pilot_url):
     """ deque to new job  of a certain pilot """
     self.resource_lock.acquire()
     #pilot_url = self.get_url(pilot_url)
     jobs = []        
     new_job_dir_url = self.get_url(pilot_url + "/new/") 
     new_job_dir = saga.advert.directory(saga.url(new_job_dir_url), 
                                         saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
     new_jobs = new_job_dir.list()
     logger.debug("Pilot Job base dir: " + new_job_dir_url + " #new jobs: " + str(len(new_jobs))
                   + " jobs: " + str(new_jobs));
     if len(new_jobs)>=1:
         job_entry=new_jobs[0]     
         job_dir_url = self.get_url(pilot_url + "/new/" + "/" + job_entry.get_string())       
         logger.debug("Open job at " + str(job_dir_url))
         job_dir = saga.advert.directory(saga.url(job_dir_url), 
                                    saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
         
         #new_job_dir.open_dir(job_entry)                        
         job_url = job_dir.get_attribute("joburl")
         #remove old job entry
         job_dir.remove(self.__remove_dbtype(job_dir_url), saga.name_space.Recursive)
             
         logger.debug("Dequeued new job: " + str(job_url))
         self.resource_lock.release()
         return self.__remove_dbtype(job_url)
     else:
         self.resource_lock.release()
         time.sleep(1)
         return 

 def file_stage_out_with_saga(self, file_list, local_dir, remote_url_prefix, remote_dir):
     for ifile in file_list:
         try:
             source_url = saga.url(remote_url_prefix)
             source_url.path= os.path.join(remote_dir, ifile)
             dest_url = saga.url("file:///" + local_dir + "/" + ifile)
             #dest_url.path = ifile
             print "(DEBUG) Staging out output.txt file at %s to %s"%(source_url.get_string(), dest_url.get_string())
             sagafile = saga.filesystem.file(source_url)
             sagafile.copy(dest_url)
         except saga.exception, e:
             error_msg = "File stage out failed: "+ source_url.get_string()

def remote_filecopy_with_saga(filename_with_local_path_from, machine_url_from, filename_with_local_path_to, machine_url_to):
    source_url = saga.url('file://' + machine_url_from + filename_with_local_path_from)
    dest_url = saga.url('gridftp://' + machine_url_to + filename_with_local_path_to)

    sagafile = saga.file.file(source_url)
    try:
        sagafile.copy(dest_url)
        print "\n(DEBUG)  remote file copy from %s of %s to %s of %s is attempted"%(filename_with_local_path_from, machine_url_from, filename_with_local_path_to, machine_url_to)
        
    except saga.exception, e:
        print "\n(WARNING) remote file copy from %s of %s to %s of %s is failed"%(filename_with_local_path_from, machine_url_from, filename_with_local_path_to, machine_url_to)
        return "Failed"

 def chunk_size(self,chunksize):
     self.__chunksize = chunksize
     chunk_list=[]
     group_chunks={}
     input=saga.url(self.__input_dir).path
     temp=saga.url(self.__tmp_dir).path
     dirList=os.listdir(input)
     for fname in dirList:
         os.system("cd " + temp + "; split -d -a 5 -b " + str(chunksize) + " " +  input + "/" + fname + " " + fname + "--" )
     dirList=os.listdir(temp)
     for fname in dirList:
         chunk_list.append([temp + "/" + fname])
     return chunk_list

def copy_with_saga(i):
    if i<RPB:
      os.system("cp "+ WORK_DIR + "/NPT.conf " + WORK_DIR + "agent/" + str(i) + "/NPT.conf")
     # source_url = saga.url('file://' + WORK_DIR + 'NPT.conf')
     # dest_url = saga.url('file://' + WORK_DIR + 'agent/' + str(i) + '/')
    elif (i>=RPB and i<(2*RPB)):
      source_url = saga.url('file://' + WORK_DIR + 'NPT.conf')
      dest_url = saga.url('gridftp://' + REMOTE1 + WORK_DIR+'agent/'+str(i)+'/')
      sagafile = saga.filesystem.file(source_url)
      try:
        sagafile.copy(dest_url)
      except saga.exception, e:
        print "\n(ERROR) remote ###NPT.CONF####file copy from %s to %s failed"%(HOST, REMOTE1)

def stage_ifiles(i):
    if not i % 2:
        try:
            os.mkdir(WORK_DIR + "agent/" + str(i))
        except OSError:
            pass
        for ifile in os.listdir(REPLICA_DIR):
            source_url = saga.url("file://" + REPLICA_DIR + ifile)
            dest_url = saga.url("file://" + WORK_DIR + "agent/" + str(i) + "/")
            sagafile = saga.filesystem.file(source_url)
            try:
                sagafile.copy(dest_url)
            except saga.exception, e:
                print str(e) + "\n(ERROR) local file ####STAGING### copy from %s to %s failed" % (REPLICA_DIR, HOST)

def copy_with_saga(i):
   # print "####################start time(npt.conf copy)" + time.asctime(time.localtime(time.time())) + "##################"
    start = time.time()
    if i<RPB:
      os.system("cp "+ WORK_DIR +"agent/"+ str(replica_id)+ "/NPT.conf " + WORK_DIR + "agent/" + str(i) + "/NPT.conf")
     # source_url = saga.url('file://' + WORK_DIR + 'NPT.conf')
     # dest_url = saga.url('file://' + WORK_DIR + 'agent/' + str(i) + '/')
    elif (i>=RPB and i<(2*RPB)):
      source_url = saga.url('file://' + WORK_DIR + 'NPT.conf')
      dest_url = saga.url('gridftp://' +"gridftp.ranger.tacc.teragrid.org:2811" + WORK_DIR1+'agent/'+str(i)+'/')
      sagafile = saga.filesystem.file(source_url)
      try:
        sagafile.copy(dest_url)
      except saga.exception, e:
        print "\n(ERROR) remote ###NPT.CONF####file copy from %s to %s failed"%(HOST, REMOTE1)

 def test_file_move(self):
     """
     Test for the saga.filesystem.move() API call 
     http://saga.cct.lsu.edu/python/apidoc/saga.filesystem._file.html#file-move
     """
     try:
         source_url = saga.url("file:///"+self.filename_a)
         target_url = saga.url("file:///"+self.filename_d)
 
         my_file = saga.filesystem.file(source_url)
         my_file.move(target_url)
         my_file.close()
             
     except saga.exception, e: 
         self.fail(e.get_full_message())

def m_dirs():
    global i_fsize , c_size , s_out, e_time, nbr_maps, nbr_reduces, app_url, app_dir, qtimes,b_uuid
    global database_host
    
    for m,q in machine.iteritems():    
        pilot_url = saga.url("advert://" + database_host + "/"+APPLICATION_NAME + "-" + str(b_uuid) + "/" + m + "/" )
        pilot_dir = saga.advert.directory(pilot_url, saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
        pilot_dir.set_attribute("state", "Unknown")
        print " created pilot directory " + str(app_url) + " with state Unknown"
    
        new_url = saga.url("advert://" + database_host + "/"+APPLICATION_NAME + "-" + str(b_uuid) + "/" + m + "/" + "new")
        new_dir = saga.advert.directory(new_url, saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
        print " created  new directory " + str(new_url) 
    for i in range(int(q)):
        pilot_dir.set_attribute("state", "Unknown")
    pilot_dir.set_attribute("state", "Running")

    def submit_job(self, glidin_url, jd):
        """ submit job via advert service to NAMD-Launcher 
            dest_url - url reference to advert job or host on which the advert job is going to run"""
        print "submit job: " + str(glidin_url)
        if self.job_url==None:
            self.job_url=self.get_job_url(glidin_url)

        for i in range(0,3):
            try:
                print "create job entry "
                self.job_dir = saga.advert.directory(saga.url(self.job_url), 
                                             saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
                print "initialized advert directory for job: " + self.job_url
                # put job description attributes to advert
                attributes = jd.list_attributes()                
                for i in attributes:          
                        if jd.attribute_is_vector(i):
                            self.job_dir.set_vector_attribute(i, jd.get_vector_attribute(i))
                        else:
                            logging.debug("Add attribute: " + str(i) + " Value: " + jd.get_attribute(i))
                            self.job_dir.set_attribute(i, jd.get_attribute(i))

                self.job_dir.set_attribute("state", str(saga.job.Unknown))
                # return self object for get_state() query    
                #logging.debug("Submission time (time to create advert entries): " + str(time.time()-start) + " s")
                return self    
            except:
                traceback.print_exc(file=sys.stdout)

 def set_job_state(self, job_url, new_state):   
     self.resource_lock.acquire()     
     job_url = self.get_url(job_url)
     logger.debug("Set state of job: " + str(job_url) + " to: " + str(new_state))
     job_dir = saga.advert.directory(saga.url(job_url), saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
     job_dir.set_attribute("state", str(new_state))
     self.resource_lock.release()

 def get_jobs_of_pilot(self, pilot_url):
     pilot_url = self.get_url(pilot_url + "/jobs")
     """ returns array of job_url that are associated with a pilot """
     pilot_dir = saga.advert.directory(saga.url(pilot_url), saga.advert.Create | saga.advert.CreateParents | saga.advert.ReadWrite)
     jobs = pilot_dir.list()   
     j = [self.__remove_dbtype(pilot_url) + "/" + i.get_string() for i in jobs]
     return j