Python PilotComputeService.create_pilot方法代码示例

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
 def submit_pilot_by_description(self, coordination_url="redis://localhost/", pilot_compute_description={}):
     pilot_compute_service = PilotComputeService(coordination_url=coordination_url)
     pilot_compute = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description)
     pilot_url = pilot_compute.get_url()
     self.pilots.append(pilot_url)
     print("Started Pilot: %s"%(pilot_url))
     self.__persist()

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
def start_run_pilot(pilot_id, coordination_url=COORD_URL):
    pilot = DareBigJobPilot.objects.get(id=pilot_id)
    pilot_compute_service = PilotComputeService(coordination_url=COORD_URL)
    print pilot.get_pilot_info()
    pilot_compute = pilot_compute_service.create_pilot(pilot_compute_description=pilot.get_pilot_info())
    pilot.pilot_url = pilot_compute.get_url()
    pilot.status = "Submitted"
    pilot.save()
    print("Started Pilot: %s " % (pilot.pilot_url), pilot.id)

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
def start_pilotcompute(pilot_compute_description=None):
    pilot_compute_service = PilotComputeService(coordination_url=COORDINATION_URL)
    if pilot_compute_description==None:
        pilot_compute_description = {
            "service_url": 'fork://localhost',
            "number_of_processes": 2,
            "working_directory": os.getcwd() + "/work/",
        }
    pilotcompute = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description)
    return pilotcompute

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
class simple:
	def __init__(self,no_jobs,pilot,COORD_url=None):
		self.no_jobs = no_jobs
		self.pilot = pilot
		if(COORD_url == None):
			self.COORD = "redis://[email protected]:6379"
		else:
			self.COORD = COORD_url
		
	def check(self):
		print 'Checkup time'
		print self.COORD
		
	def startpilot(self):
		print 'Start pilot service'
		self.pilot_compute_service = PilotComputeService(self.COORD)
		self.pilot_compute_description = { 
						"service_url" : self.pilot["service_url"]
						}
		if self.pilot.has_key("number_of_processes"):
			self.pilot_compute_description["number_of_processes"] = self.pilot["number_of_processes"] 
		if self.pilot.has_key("working_directory"):
			self.pilot_compute_description["working_directory"] = self.pilot["working_directory"]
		if self.pilot.has_key("queue"):
			self.pilot_compute_description["queue"] = self.pilot["queue"]
		if self.pilot.has_key("walltime"):
			self.pilot_compute_description["walltime"] = self.pilot["walltime"]
		self.pilotjob = self.pilot_compute_service.create_pilot(pilot_compute_description=self.pilot_compute_description)
		print 'Pilot successfully started'
	
	def startCU(self):
		print 'Starting Compute Unit submissions'
		self.compute_data_service = ComputeDataService()
		self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
		for i in range(self.no_jobs):
			print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"])
			self.compute_unit_description = {
							"executable":"/bin/echo",
							"arguments" : ["$MYOUTPUT"],
							"environment" : {'MYOUTPUT':'"Hello from Simple API"'},
							"number_of_processes" : 1,
							"output" : "stdout.txt",
							"error" : "stderr.txt"
							}
			self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description)
		print 'All Compute Units Submitted. Waiting for completion'
		self.compute_data_service.wait()
		print 'All CU executions completed'
		
	def terminate(self):
		print 'Terminating pilot'
		self.compute_data_service.cancel()
		self.pilot_compute_service.cancel()

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
        RUN_ID = sys.argv[1]
    pilot_compute_service = PilotComputeService(COORDINATION_URL)

    # create pilot job service
    pilot_compute_description = {
         "service_url": 'pbs+ssh://trestles.sdsc.edu',
         "number_of_processes": 1,
         "queue": "shared",
         "project": "TG-MCB090174",
         "working_directory": "/home/jweichel/agent",
         "walltime": 90
    #     'affinity_datacenter_label': "trestles",
    #     'affinity_machine_label': "mymachine"
                                }
    # initiate a pilot job
    pilotjob = pilot_compute_service.create_pilot(pilot_compute_description)

    # create pilot data service (factory for data pilots (physical,distributed storage))
    # and pilot data where data is to be moved
    #pilot_data_service = PilotDataService(COORDINATION_URL)
    #pilot_data_description = {
    #     "service_url":"ssh://localhost/"+os.getenv("HOME")+"/matrix/pilotdata",
    #     "size": 1,
    #     "affinity_datacenter_label":"trestles",
    #     "affinity_machine_label":"mymachine"
    #                         }

    #ps = pilot_data_service.create_pilot(pilot_data_description=pilot_data_description)

    #compute_data_service = ComputeDataService()
    #compute_data_service.add_pilot_compute_service(pilot_compute_service)

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]

#.........这里部分代码省略.........
        # cancel all not-running submitted subjobs
        #        for k in range(self.nreplicas):
        #            if self.status[k]['running_status'] == "R":
        #                if self.cus[k].get_state() != "Running":
        #                    self.cus[k].cancel()
        #                    self.status[k]['running_status'] = "W"
        # update status
        #        self.updateStatus()
        #        self.print_status()
        # wait until running jobs complete
        self.cds.wait()

    def cleanJob(self):
        self.cds.cancel()
        self.pj.cancel()

    def launch_pilotjob(self):
        # pilotjob: PilotJob description
        # pilotjob: Variables defined in command.inp
        pcd = {
            "service_url": self.keywords.get("RESOURCE_URL"),
            "number_of_processes": self.keywords.get("TOTAL_CORES"),
            "working_directory": self.bj_working_dir,
            "queue": self.keywords.get("QUEUE"),
            "processes_per_node": self.ppn,
            "project": self.keywords.get("PROJECT"),
            "walltime": int(self.keywords.get("WALL_TIME")),
        }

        if self.keywords.get("SGE_WAYNESS") is not None:
            pcd["spmd_variation"] = self.keywords.get("SGE_WAYNESS")

        # pilotjob: Create pilot job with above description
        self.pj.create_pilot(pilot_compute_description=pcd)
        self.cds.add_pilot_compute_service(self.pj)
        self.pilotcompute = self.pj.list_pilots()[0]

    def _write_status(self):
        """
        Pickle the current state of the RE job and write to in BASENAME.stat. 
        """
        status_file = "%s.stat" % self.basename
        f = _open(status_file, "w")
        pickle.dump(self.status, f)
        f.close()

    def _read_status(self):
        """
        Unpickle and load the current state of the RE job from BASENAME.stat.
        """
        status_file = "%s.stat" % self.basename
        f = _open(status_file, "r")
        self.status = pickle.load(f)
        f.close()

    def print_status(self):
        """
        Writes to BASENAME_stat.txt a text version of the status of the RE job. 
        It's fun to follow the progress in real time by doing:
        watch cat BASENAME_stat.txt
        """
        log = "Replica  State  Status  Cycle \n"
        for k in range(self.nreplicas):
            log += "%6d   %5d  %5s  %5d \n" % (
                k,
                self.status[k]["stateid_current"],

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
                          #"service_url": 'fork://localhost',
                          "number_of_processes": 1,
                          'affinity_datacenter_label': "us-east",              
                          'affinity_machine_label': "", 
                          #'working_directory': os.getcwd(),
                          # cloud specific attributes
                          "vm_id":"emi-36913A82",
                          "vm_ssh_username":"root",
                          "vm_ssh_keyname":"luckow",
                          "vm_ssh_keyfile":"/Users/luckow/.ssh/eucakey-india",
                          "vm_type":"c1.xlarge",
                          "access_key_id":"",
                          "secret_access_key":""
                         }
 
 pilotjob = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description_amazon_west)
 
 compute_data_service = ComputeDataService()
 compute_data_service.add_pilot_compute_service(pilot_compute_service)
 compute_data_service.add_pilot_data_service(pilot_data_service)
 
 # create empty data unit for output data
 output_data_unit_description = {
      "file_urls": []              
 }
 output_data_unit = pd.submit_data_unit(output_data_unit_description)
 output_data_unit.wait()
 
 # create compute unit
 compute_unit_description = {
         "executable": "/bin/cat",

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
    start_time=time.time()
    pilot_compute_service = PilotComputeService(COORDINATION_URL)
    pilot_compute_description=[]

    pilot_compute_description.append({ "service_url": "ssh://localhost",
                                       "number_of_processes": 12,
                                       "allocation": "TG-MCB090174",
	                               "queue": "normal",
                                       "processes_per_node":12,
                                       "working_directory": os.getcwd()+"/agent",
                                       "walltime":10,
                                     })

    for pcd in pilot_compute_description:
        pilot_compute_service.create_pilot(pilot_compute_description=pcd)

    compute_data_service = ComputeDataService()
    compute_data_service.add_pilot_compute_service(pilot_compute_service)

    print ("Finished Pilot-Job setup. Submit compute units")
    # submit Set A compute units
    for i in range(NUMBER_JOBS):
        compute_unit_description = {
                "executable": "/bin/echo",
                "arguments": ["Hello","$ENV1","$ENV2"],
                "environment": ['ENV1=env_arg1','ENV2=env_arg2'],
                "total_cpu_count": 4,            
                "spmd_variation":"mpi",
                "output": "A_stdout.txt",
                "error": "A_stderr.txt",

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
class simple:
	#Input values from user - Number of jobs, Pilot Description, Redis Server Coordination URL
	def __init__(self,no_jobs,pilot,COORD_url=None):
		self.no_jobs = no_jobs
		self.pilot = pilot
		if(COORD_url == None):
			self.COORD = "redis://[email protected]:6379"
		else:
			self.COORD = COORD_url
		#Time variable to analyse timing responses
		self.pilot_setup_time = 0
		self.total_cu_time = 0
		self.cu_sub_time = 0
		self.cu_wait_time = 0
			
	def startpilot(self):
		#API currently supports single pilot applications
		print 'Start pilot service'
		p1=time.time()
		self.pilot_compute_service = PilotComputeService(self.COORD)
		#Mandatory service url,number_of_processes of the Pilot
		self.pilot_compute_description = { 
						"service_url" : self.pilot["service_url"],
						"number_of_processes" : self.pilot["number_of_processes"]
						}
		#Check for possible keys for Working Directory,
		#Queue, Walltime. Take default values if not mentioned.
		if self.pilot.has_key("working_directory"):
			self.pilot_compute_description["working_directory"] = self.pilot["working_directory"]
		if self.pilot.has_key("queue"):
			self.pilot_compute_description["queue"] = self.pilot["queue"]
		if self.pilot.has_key("walltime"):
			self.pilot_compute_description["walltime"] = self.pilot["walltime"]
		self.pilotjob = self.pilot_compute_service.create_pilot(pilot_compute_description=self.pilot_compute_description)
		p2=time.time()
		self.pilot_setup_time = p2 - p1
		print 'Pilot successfully started'
	
	def startCU(self):
		print 'Starting Compute Unit submissions'
		#p1,p2,p3,p4,p5 - Probes for time calculations
		p1 = time.time()
		self.compute_data_service = ComputeDataService()
		self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
		for i in range(self.no_jobs):
			print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"])
			p2 = time.time()
			self.compute_unit_description = {
							"executable":"/bin/sleep 4",
							#"arguments" : ["$MYOUTPUT"],
							#"environment" : {'MYOUTPUT':'"Hello from Simple API"'},
							"number_of_processes" : 1,
							"output" : "stdout.txt",
							"error" : "stderr.txt"
							}
			self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description)
			p3 = time.time()
			self.cu_sub_time = self.cu_sub_time + (p3-p2)
		print 'All Compute Units Submitted. Waiting for completion'
		p4 = time.time()
		self.compute_data_service.wait()
		p5 = time.time()
		self.total_cu_time = p5 - p1
		self.cu_wait_time = p5 - p4
		print 'All CU executions completed'
		
	def terminate(self):
		#Terminate all CUs and pilot, Display all timing responses
		print 'Terminating pilot'
		self.compute_data_service.cancel()
		self.pilot_compute_service.cancel()
		print 'No of jobs : ',self.no_jobs
		print 'Pilot setup time : ',self.pilot_setup_time
		print 'CU submission time : ',self.cu_sub_time
		print 'CU wait time : ',self.cu_wait_time
		print 'Total CU time : ',self.total_cu_time

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
                          #"service_url": 'fork://localhost',
                          "number_of_processes": 1,
                          'affinity_datacenter_label': "us-east",              
                          'affinity_machine_label': "", 
                          #'working_directory': os.getcwd(),
                          # cloud specific attributes
                          "vm_id":"emi-36913A82",
                          "vm_ssh_username":"root",
                          "vm_ssh_keyname":"luckow",
                          "vm_ssh_keyfile":"/Users/luckow/.ssh/eucakey-india",
                          "vm_type":"c1.xlarge",
                          "access_key_id":"8MCXRAMXMHDYKWNKXZ8WF",
                          "secret_access_key":"YrcUqSw2Arxshrh3ZtenkxerWwCWdMTKvZYoLPAo"
                         }
 
 pilotjob = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description_euca_india)
 
 compute_data_service = ComputeDataService()
 compute_data_service.add_pilot_compute_service(pilot_compute_service)
 compute_data_service.add_pilot_data_service(pilot_data_service)
 
 # create empty data unit for output data
 output_data_unit_description = {
      "file_urls": []              
 }
 output_data_unit = pd.submit_data_unit(output_data_unit_description)
 output_data_unit.wait()
 
 # create compute unit
 compute_unit_description = {
         "executable": "/bin/cat",

# 需要导入模块: from pilot import PilotComputeService [as 别名]
# 或者: from pilot.PilotComputeService import create_pilot [as 别名]
class DareManager(object):
    """DARE manager:
       - reads different configuration files
       - submits compute/data units as that in various steps"""

    """Constructor"""
    def __init__(self, conffile="/path/to/conf/file"):
        "" ""
        self.dare_conffile = conffile
        self.workflow = PrepareWorkFlow(self.dare_conffile)
        self.updater = Updater(self.workflow.update_site_db, self.workflow.dare_web_id)
        self.dare_id = "dare-" + str(uuid.uuid1())
        self.data_pilot_service_repo = []
        self.step_threads = {}
        try:
            self.start()
        except KeyboardInterrupt:
            self.quit(message='KeyboardInterrupt')

    def start(self):
        darelogger.info("Creating Compute Engine service ")
        self.pilot_compute_service = PilotComputeService(coordination_url=COORDINATION_URL)
        self.pilot_data_service = PilotDataService(coordination_url=COORDINATION_URL)

        for compute_pilot, desc in self.workflow.compute_pilot_repo.items():
            self.pilot_compute_service.create_pilot(pilot_compute_description=desc)

        for data_pilot, desc in self.workflow.data_pilot_repo.items():
            self.data_pilot_service_repo.append(self.pilot_data_service.create_pilot(pilot_data_description=desc))

        self.compute_data_service = ComputeDataServiceDecentral()
        self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
        self.compute_data_service.add_pilot_data_service(self.pilot_data_service)

        ### run the steps
        self.step_start_lock = threading.RLock()
        self.step_run_lock = threading.RLock()

        for step_id in self.workflow.step_units_repo.keys():
                darelogger.info(" Sumitted step %s " % step_id)
                self.step_start_lock.acquire()
                self.start_thread_step_id = step_id
                self.step_start_lock.release()
                self.step_threads[step_id] = threading.Thread(target=self.start_step)
                self.step_threads[step_id].start()

        while(1):
            count_step = [v.is_alive() for k, v in self.step_threads.items()]
            darelogger.info('count_step %s' % count_step)
            if not True in count_step and len(count_step) > 0:
                break
            time.sleep(10)

        darelogger.info(" All Steps Done processing")

        self.quit(message='quit gracefully')

    def check_to_start_step(self, step_id):
        flags = []
        darelogger.info(self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps'])
        if self.workflow.step_units_repo[step_id].get_status() == StepUnitStates.New:
            for dep_step_id in self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps']:
                if self.workflow.step_units_repo[dep_step_id].get_status() != StepUnitStates.Done:
                    flags.append(False)
                darelogger.info(self.workflow.step_units_repo[dep_step_id].get_status())
        return False if False in flags else True

    def start_step(self):
        self.step_start_lock.acquire()
        step_id = self.start_thread_step_id
        self.step_start_lock.release()

        while(1):
            darelogger.info(" Checking to start step %s " % step_id)
            if self.check_to_start_step(step_id):
                self.run_step(step_id)
                break
            else:
                darelogger.info(" Cannot start this step %s sleeping..." % step_id)
                time.sleep(10)

    def run_step(self, step_id):
        #self.step_run_lock.acquire()
        #job started update status
        this_su = self.workflow.step_units_repo[step_id].UnitInfo
        self.updater.update_status(this_su['dare_web_id'], "%s in step %s" % ('Running',  this_su['name']))

        darelogger.info(" Started running %s " % step_id)

        jobs = []
        job_start_times = {}
        job_states = {}
        NUMBER_JOBS = len(self.workflow.step_units_repo[step_id].UnitInfo['compute_units'])
        for cu_id in self.workflow.step_units_repo[step_id].UnitInfo['compute_units']:
            compute_unit_desc = self.workflow.compute_units_repo[cu_id]
            input_dus = compute_unit_desc.pop('input_data_units')
            output_dus = compute_unit_desc.pop('output_data_units')
            input_data_units = []
            for du_id in input_dus:
                input_data_units.append(self.compute_data_service.submit_data_unit(self.workflow.data_units_repo[du_id]))
#.........这里部分代码省略.........