Python Pipeline.restart方法代码示例

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import restart [as 别名]
class AbstractApplication(object):
    """Framework class for writing applications for PydPiper. 
    
       This class defines the default behaviour for accepting common command-line options, and executing the application
       under various queueing systems. 
       
       Subclasses should extend the following methods:
           setup_appName()
           setup_logger() [optional, default method is defined here]
           setup_options()
           run()
    
       Usage: 
          class MyApplication(AbstractApplication):
                ... 
           
          if __name__ == "__main__":
              application = ConcreteApplication()
              application.start()
    """
    def __init__(self):
        Pyro.config.PYRO_MOBILE_CODE=1 
        self.parser = MyParser()
    
    def _setup_options(self):
            # PydPiper options
        basic_group = OptionGroup(self.parser,  "Basic execution control",
                                  "Options controlling how and where the code is run.")
        basic_group.add_option("--uri-file", dest="urifile",
                               type="string", default=None,
                               help="Location for uri file if NameServer is not used. If not specified, default is current working directory.")
        basic_group.add_option("--use-ns", dest="use_ns",
                               action="store_true",
                               help="Use the Pyro NameServer to store object locations")
        basic_group.add_option("--create-graph", dest="create_graph",
                               action="store_true", default=False,
                               help="Create a .dot file with graphical representation of pipeline relationships [default = %default]")
        basic_group.add_option("--num-executors", dest="num_exec", 
                               type="int", default=0, 
                               help="Launch executors automatically without having to run pipeline_excutor.py independently.")
        basic_group.add_option("--time", dest="time", 
                               type="string", default="2:00:00:00", 
                               help="Wall time to request for each executor in the format dd:hh:mm:ss")
        basic_group.add_option("--proc", dest="proc", 
                               type="int", default=8,
                               help="Number of processes per executor. Default is 8. Also sets max value for processor use per executor.")
        basic_group.add_option("--mem", dest="mem", 
                               type="float", default=16,
                               help="Total amount of requested memory. Default is 16G.")
        basic_group.add_option("--ppn", dest="ppn", 
                               type="int", default=8,
                               help="Number of processes per node. Default is 8. Used when --queue=pbs")
        basic_group.add_option("--queue", dest="queue", 
                               type="string", default=None,
                               help="Use specified queueing system to submit jobs. Default is None.")
        basic_group.add_option("--sge-queue-opts", dest="sge_queue_opts", 
                               type="string", default=None,
                               help="For --queue=sge, allows you to specify different queues. If not specified, default is used.")
        basic_group.add_option("--restart", dest="restart", 
                               action="store_true",
                               help="Restart pipeline using backup files.")
        basic_group.add_option("--output-dir", dest="output_directory",
                               type="string", default=None,
                               help="Directory where output data and backups will be saved.")
        self.parser.set_defaults(execute=True)
        basic_group.add_option("--execute", dest="execute",
                               action="store_true",
                               help="Actually execute the planned commands [default]")
        basic_group.add_option("--no-execute", dest="execute",
                               action="store_false",
                               help="Opposite of --execute")
        self.parser.add_option_group(basic_group)
    
    def _setup_pipeline(self):
        self.pipeline = Pipeline()
        
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)
    
    def reconstructCommand(self):    
        reconstruct = ""
        for i in range(len(sys.argv)):
            reconstruct += sys.argv[i] + " "
        logger.info("Command is: " + reconstruct)
        
    def start(self):
        self._setup_options()
        self.setup_options()
        
        self.options, self.args = self.parser.parse_args()        
        self._setup_pipeline()
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()
#.........这里部分代码省略.........