Python Pipeline.initialize方法代码示例

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import initialize [as 别名]
class AbstractApplication(object):
    """Framework class for writing applications for PydPiper. 
    
       This class defines the default behaviour for accepting common command-line options, and executing the application
       under various queueing systems. 
       
       Subclasses should extend the following methods:
           setup_appName()
           setup_logger() [optional, default method is defined here]
           setup_options()
           run()
    
       Usage: 
          class MyApplication(AbstractApplication):
                ... 
           
          if __name__ == "__main__":
              application = MyApplication()
              application.start()
    """
    def __init__(self):
        self.parser = MyParser()
        self.__version__ = get_distribution("pydpiper").version
    
    def _setup_options(self):
            # PydPiper options
        addExecutorOptionGroup(self.parser)
        addApplicationOptionGroup(self.parser)
    
    def _print_version(self):
        if self.options.show_version:
            print self.__version__
            sys.exit()
    
    def _setup_pipeline(self):
        self.pipeline = Pipeline()
        
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)
    
    def reconstructCommand(self):    
        reconstruct = ""
        for i in range(len(sys.argv)):
            reconstruct += sys.argv[i] + " "
        logger.info("Command is: " + reconstruct)
        
    def start(self):
        self._setup_options()
        self.setup_options()
        
        self.options, self.args = self.parser.parse_args()
        
        self._print_version()   
        
        #Check to make sure some executors have been specified. 
        noExecSpecified(self.options.num_exec)
             
        self._setup_pipeline()
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()
        
        if self.options.queue=="pbs":
            roq = runOnQueueingSystem(self.options, sys.argv)
            roq.createPbsScripts()
            return 
        
        if self.options.restart:
            print "\nThe restart option is deprecated (pipelines are not pickled anymore, because it takes too much time). Will restart based on which files exists already\n"
            #logger.info("Restarting pipeline from pickled files.")
            #self.pipeline.restart()
            self.reconstructCommand()
            self.run()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
        else:
            self.reconstructCommand()
            self.run()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
                            
        if self.options.create_graph:
            logger.debug("Writing dot file...")
            nx.write_dot(self.pipeline.G, "labeled-tree.dot")
            logger.debug("Done.")
                
        if not self.options.execute:
            print "Not executing the command (--no-execute is specified).\nDone."
            return
        
        #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
        # if use_ns is specified, Pyro NameServer must be started. 
        logger.info("Starting pipeline daemon...")
        pipelineDaemon(self.pipeline, self.options, sys.argv[0])
#.........这里部分代码省略.........

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import initialize [as 别名]

#.........这里部分代码省略.........
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)

    def reconstructCommand(self):    
        reconstruct = ' '.join(sys.argv)
        logger.info("Command is: " + reconstruct)
        logger.info("Command version : " + self.__version__)
        # also, because this is probably a better file for it (also has similar
        # naming conventions as the pipeline-stages.txt file:
        fileForCommandAndVersion = self.options.pipeline_name + "-command-and-version-" + time.strftime("%d-%m-%Y-at-%H-%m-%S") + ".sh"
        pf = open(fileForCommandAndVersion, "w")
        pf.write("#!/usr/bin/env bash\n")
        pf.write("# Command version is: " + self.__version__ + "\n")
        pf.write("# Command was: \n")
        pf.write(reconstruct + '\n')
        pf.write("# options were: \n# %s" % self.options)
        pf.close()
        
    def start(self):
        logger.info("Calling `start`")
        self._setup_options()
        self.setup_options()
        self.options = self.parser.parse_args()
        self.args = self.options.files

        self._print_version()

        # Check to make sure some executors have been specified if we are 
        # actually going to run:
        if self.options.execute:
            noExecSpecified(self.options.num_exec)
             
        self._setup_pipeline(self.options)
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()

        # TODO this doesn't capture environment variables
        # or contents of any config file so isn't really complete
        self.reconstructCommand()

        pbs_submit = self.options.queue_type == "pbs" \
                     and not self.options.local

        # --create-graph causes the pipeline to be constructed
        # both at PBS submit time and on the grid; this may be an extremely
        # expensive duplication
        if (self.options.execute and not pbs_submit) or self.options.create_graph:
            logger.debug("Calling `run`")
            self.run()
            logger.debug("Calling `initialize`")
            self.pipeline.initialize()
            self.pipeline.printStages(self.options.pipeline_name)

        if self.options.create_graph:
            logger.debug("Writing dot file...")
            nx.write_dot(self.pipeline.G, str(self.options.pipeline_name) + "_labeled-tree.dot")
            logger.debug("Done.")

        if not self.options.execute:
            print("Not executing the command (--no-execute is specified).\nDone.")
            return
        
        if pbs_submit:
            roq = runOnQueueingSystem(self.options, sys.argv)
            roq.createAndSubmitPbsScripts()
            logger.info("Finished submitting PBS job scripts...quitting")
            return 
                
        #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
        # if use_ns is specified, Pyro NameServer must be started. 
        logger.info("Starting pipeline daemon...")
        pipelineDaemon(self.pipeline, self.options, sys.argv[0])
        logger.info("Server has stopped.  Quitting...")

    def setup_appName(self):
        """sets the name of the application"""
        pass

    def setup_logger(self):
        """sets logging info specific to application"""
        FORMAT = '%(asctime)-15s %(name)s %(levelname)s %(process)d/%(threadName)s: %(message)s'
        now = datetime.now().strftime("%Y-%m-%d-at-%H:%M:%S")
        FILENAME = str(self.appName) + "-" + now + '-pid-' + str(os.getpid())  + ".log"
        logging.basicConfig(filename=FILENAME, format=FORMAT, level=logging.DEBUG)

    def setup_options(self):
        """Set up the self.options option parser with options this application needs."""
        pass
    
    def run(self):
        """Run this application.
        
           """
        pass

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import initialize [as 别名]

#.........这里部分代码省略.........
        basic_group.add_option("--queue", dest="queue", 
                               type="string", default=None,
                               help="Use specified queueing system to submit jobs. Default is None.")
        basic_group.add_option("--sge-queue-opts", dest="sge_queue_opts", 
                               type="string", default=None,
                               help="For --queue=sge, allows you to specify different queues. If not specified, default is used.")
        basic_group.add_option("--restart", dest="restart", 
                               action="store_true",
                               help="Restart pipeline using backup files.")
        basic_group.add_option("--output-dir", dest="output_directory",
                               type="string", default=None,
                               help="Directory where output data and backups will be saved.")
        self.parser.set_defaults(execute=True)
        basic_group.add_option("--execute", dest="execute",
                               action="store_true",
                               help="Actually execute the planned commands [default]")
        basic_group.add_option("--no-execute", dest="execute",
                               action="store_false",
                               help="Opposite of --execute")
        self.parser.add_option_group(basic_group)
    
    def _setup_pipeline(self):
        self.pipeline = Pipeline()
        
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)
    
    def reconstructCommand(self):    
        reconstruct = ""
        for i in range(len(sys.argv)):
            reconstruct += sys.argv[i] + " "
        logger.info("Command is: " + reconstruct)
        
    def start(self):
        self._setup_options()
        self.setup_options()
        
        self.options, self.args = self.parser.parse_args()        
        self._setup_pipeline()
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()
        
        if self.options.queue=="pbs":
            roq = runOnQueueingSystem(self.options, sys.argv)
            roq.createPbsScripts()
            return 
        
        if self.options.restart:
            logger.info("Restarting pipeline from pickled files.")
            self.pipeline.restart()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
        else:
            self.reconstructCommand()
            self.run()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
                            
        if self.options.create_graph:
            logger.debug("Writing dot file...")
            nx.write_dot(self.pipeline.G, "labeled-tree.dot")
            logger.debug("Done.")
                
        if not self.options.execute:
            print "Not executing the command (--no-execute is specified).\nDone."
            return
        
        #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
        # if use_ns is specified, Pyro NameServer must be started. 
        logger.info("Starting pipeline daemon...")
        pipelineDaemon(self.pipeline, self.options, sys.argv[0])
        logger.info("Server has stopped.  Quitting...")

    def setup_appName(self):
        """sets the name of the application"""
        pass

    def setup_logger(self):
        """sets logging info specific to application"""
        FORMAT = '%(asctime)-15s %(name)s %(levelname)s: %(message)s'
        now = datetime.now()  
        FILENAME = str(self.appName) + "-" + now.strftime("%Y%m%d-%H%M%S%f") + ".log"
        logging.basicConfig(filename=FILENAME, format=FORMAT, level=logging.DEBUG)

    def setup_options(self):
        """Set up the self.options option parser with options this application needs."""
        pass
    
    def run(self):
        """Run this application.
        
           """
        pass