Python Pipeline.setBackupFileLocation方法代码示例

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import setBackupFileLocation [as 别名]
class AbstractApplication(object):
    """Framework class for writing applications for PydPiper. 
    
       This class defines the default behaviour for accepting common command-line options, and executing the application
       under various queueing systems. 
       
       Subclasses should extend the following methods:
           setup_appName()
           setup_logger() [optional, default method is defined here]
           setup_options()
           run()
    
       Usage: 
          class MyApplication(AbstractApplication):
                ... 
           
          if __name__ == "__main__":
              application = MyApplication()
              application.start()
    """
    def __init__(self):
        self.parser = MyParser()
        self.__version__ = get_distribution("pydpiper").version
    
    def _setup_options(self):
            # PydPiper options
        addExecutorOptionGroup(self.parser)
        addApplicationOptionGroup(self.parser)
    
    def _print_version(self):
        if self.options.show_version:
            print self.__version__
            sys.exit()
    
    def _setup_pipeline(self):
        self.pipeline = Pipeline()
        
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)
    
    def reconstructCommand(self):    
        reconstruct = ""
        for i in range(len(sys.argv)):
            reconstruct += sys.argv[i] + " "
        logger.info("Command is: " + reconstruct)
        
    def start(self):
        self._setup_options()
        self.setup_options()
        
        self.options, self.args = self.parser.parse_args()
        
        self._print_version()   
        
        #Check to make sure some executors have been specified. 
        noExecSpecified(self.options.num_exec)
             
        self._setup_pipeline()
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()
        
        if self.options.queue=="pbs":
            roq = runOnQueueingSystem(self.options, sys.argv)
            roq.createPbsScripts()
            return 
        
        if self.options.restart:
            print "\nThe restart option is deprecated (pipelines are not pickled anymore, because it takes too much time). Will restart based on which files exists already\n"
            #logger.info("Restarting pipeline from pickled files.")
            #self.pipeline.restart()
            self.reconstructCommand()
            self.run()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
        else:
            self.reconstructCommand()
            self.run()
            self.pipeline.initialize()
            self.pipeline.printStages(self.appName)
                            
        if self.options.create_graph:
            logger.debug("Writing dot file...")
            nx.write_dot(self.pipeline.G, "labeled-tree.dot")
            logger.debug("Done.")
                
        if not self.options.execute:
            print "Not executing the command (--no-execute is specified).\nDone."
            return
        
        #pipelineDaemon runs pipeline, launches Pyro client/server and executors (if specified)
        # if use_ns is specified, Pyro NameServer must be started. 
        logger.info("Starting pipeline daemon...")
        pipelineDaemon(self.pipeline, self.options, sys.argv[0])
#.........这里部分代码省略.........

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import setBackupFileLocation [as 别名]
class AbstractApplication(object):
    """Framework class for writing applications for PydPiper. 
    
       This class defines the default behaviour for accepting common command-line options, and executing the application
       under various queueing systems. 
       
       Subclasses should extend the following methods:
           setup_appName()
           setup_logger() [optional, default method is defined here]
           setup_options()
           run()
    
       Usage: 
          class MyApplication(AbstractApplication):
                ... 
           
          if __name__ == "__main__":
              application = MyApplication()
              application.start()
    """
    def __init__(self):
        # use an environment variable to look for a default config file
        # Alternately, we could use a default location for the file
        # (say `files = ['/etc/pydpiper.cfg', '~/pydpiper.cfg', './pydpiper.cfg']`)
        default_config_file = os.getenv("PYDPIPER_CONFIG_FILE")
        if default_config_file is not None:
            files = [default_config_file]
        else:
            files = []
        self.parser = MyParser(default_config_files=files)
        self.__version__ = get_distribution("pydpiper").version  # pylint: disable=E1101
    
    def _setup_options(self):
            # PydPiper options
        addExecutorArgumentGroup(self.parser)
        addApplicationArgumentGroup(self.parser)
    
    def _print_version(self):
        if self.options.show_version:
            print(self.__version__)
            sys.exit()
    
    def _setup_pipeline(self, options):
        self.pipeline = Pipeline(options)

    # FIXME check that only one server is running with a given output directory
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)

    def reconstructCommand(self):    
        reconstruct = ' '.join(sys.argv)
        logger.info("Command is: " + reconstruct)
        logger.info("Command version : " + self.__version__)
        # also, because this is probably a better file for it (also has similar
        # naming conventions as the pipeline-stages.txt file:
        fileForCommandAndVersion = self.options.pipeline_name + "-command-and-version-" + time.strftime("%d-%m-%Y-at-%H-%m-%S") + ".sh"
        pf = open(fileForCommandAndVersion, "w")
        pf.write("#!/usr/bin/env bash\n")
        pf.write("# Command version is: " + self.__version__ + "\n")
        pf.write("# Command was: \n")
        pf.write(reconstruct + '\n')
        pf.write("# options were: \n# %s" % self.options)
        pf.close()
        
    def start(self):
        logger.info("Calling `start`")
        self._setup_options()
        self.setup_options()
        self.options = self.parser.parse_args()
        self.args = self.options.files

        self._print_version()

        # Check to make sure some executors have been specified if we are 
        # actually going to run:
        if self.options.execute:
            noExecSpecified(self.options.num_exec)
             
        self._setup_pipeline(self.options)
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()

        # TODO this doesn't capture environment variables
        # or contents of any config file so isn't really complete
        self.reconstructCommand()

        pbs_submit = self.options.queue_type == "pbs" \
                     and not self.options.local

        # --create-graph causes the pipeline to be constructed
        # both at PBS submit time and on the grid; this may be an extremely
        # expensive duplication
        if (self.options.execute and not pbs_submit) or self.options.create_graph:
#.........这里部分代码省略.........

# 需要导入模块: from pydpiper.pipeline import Pipeline [as 别名]
# 或者: from pydpiper.pipeline.Pipeline import setBackupFileLocation [as 别名]
class AbstractApplication(object):
    """Framework class for writing applications for PydPiper. 
    
       This class defines the default behaviour for accepting common command-line options, and executing the application
       under various queueing systems. 
       
       Subclasses should extend the following methods:
           setup_appName()
           setup_logger() [optional, default method is defined here]
           setup_options()
           run()
    
       Usage: 
          class MyApplication(AbstractApplication):
                ... 
           
          if __name__ == "__main__":
              application = ConcreteApplication()
              application.start()
    """
    def __init__(self):
        Pyro.config.PYRO_MOBILE_CODE=1 
        self.parser = MyParser()
    
    def _setup_options(self):
            # PydPiper options
        basic_group = OptionGroup(self.parser,  "Basic execution control",
                                  "Options controlling how and where the code is run.")
        basic_group.add_option("--uri-file", dest="urifile",
                               type="string", default=None,
                               help="Location for uri file if NameServer is not used. If not specified, default is current working directory.")
        basic_group.add_option("--use-ns", dest="use_ns",
                               action="store_true",
                               help="Use the Pyro NameServer to store object locations")
        basic_group.add_option("--create-graph", dest="create_graph",
                               action="store_true", default=False,
                               help="Create a .dot file with graphical representation of pipeline relationships [default = %default]")
        basic_group.add_option("--num-executors", dest="num_exec", 
                               type="int", default=0, 
                               help="Launch executors automatically without having to run pipeline_excutor.py independently.")
        basic_group.add_option("--time", dest="time", 
                               type="string", default="2:00:00:00", 
                               help="Wall time to request for each executor in the format dd:hh:mm:ss")
        basic_group.add_option("--proc", dest="proc", 
                               type="int", default=8,
                               help="Number of processes per executor. Default is 8. Also sets max value for processor use per executor.")
        basic_group.add_option("--mem", dest="mem", 
                               type="float", default=16,
                               help="Total amount of requested memory. Default is 16G.")
        basic_group.add_option("--ppn", dest="ppn", 
                               type="int", default=8,
                               help="Number of processes per node. Default is 8. Used when --queue=pbs")
        basic_group.add_option("--queue", dest="queue", 
                               type="string", default=None,
                               help="Use specified queueing system to submit jobs. Default is None.")
        basic_group.add_option("--sge-queue-opts", dest="sge_queue_opts", 
                               type="string", default=None,
                               help="For --queue=sge, allows you to specify different queues. If not specified, default is used.")
        basic_group.add_option("--restart", dest="restart", 
                               action="store_true",
                               help="Restart pipeline using backup files.")
        basic_group.add_option("--output-dir", dest="output_directory",
                               type="string", default=None,
                               help="Directory where output data and backups will be saved.")
        self.parser.set_defaults(execute=True)
        basic_group.add_option("--execute", dest="execute",
                               action="store_true",
                               help="Actually execute the planned commands [default]")
        basic_group.add_option("--no-execute", dest="execute",
                               action="store_false",
                               help="Opposite of --execute")
        self.parser.add_option_group(basic_group)
    
    def _setup_pipeline(self):
        self.pipeline = Pipeline()
        
    def _setup_directories(self):
        """Output and backup directories setup here."""
        if not self.options.output_directory:
            self.outputDir = os.getcwd()
        else:
            self.outputDir = makedirsIgnoreExisting(self.options.output_directory)
        self.pipeline.setBackupFileLocation(self.outputDir)
    
    def reconstructCommand(self):    
        reconstruct = ""
        for i in range(len(sys.argv)):
            reconstruct += sys.argv[i] + " "
        logger.info("Command is: " + reconstruct)
        
    def start(self):
        self._setup_options()
        self.setup_options()
        
        self.options, self.args = self.parser.parse_args()        
        self._setup_pipeline()
        self._setup_directories()
        
        self.appName = self.setup_appName()
        self.setup_logger()
#.........这里部分代码省略.........