Python workflow.SequentialTaskCollection类代码示例

    def __init__(self, videofile, email_from, smtp_server, **extra):
        self.videofile = videofile
        videofilename = os.path.basename(videofile)

        inboxdir = os.path.dirname(videofile)
        self.vdescrfile = os.path.join(inboxdir, 'video.description.txt')
        self.inboxdir = inboxdir
        self.email_to = ''
        self.email_from = email_from
        self.smtp_server = smtp_server
        self.video_is_needed = True

        # Look for a 'configuration' file in the inboxdir to override
        # bemovi parameters

        videoname = videofilename.rsplit('.', 1)[0]

        self.extra = extra
        extra['jobname'] = 'BemoviWorkflow_%s' % videoname
        extra['videoname'] = videoname

        extra['base_output_dir'] = os.path.join(
            inboxdir + '.out',
            extra['jobname'])

        self.update_configuration()

        plocator = ParticleLocator(videofile, **extra)
        plinker = ParticleLinker(
            os.path.join(extra['base_output_dir'],
                         ParticleLocator.application,
                         extra['videoname'] + '.ijout.txt',),
            **extra)
        SequentialTaskCollection.__init__(self, [plocator, plinker], **extra)

    def __init__(self, grayscaled_image, output_dir, output_file,
                 colors, warhol_dir, **extra_args):
        self.grayscaled_image = grayscaled_image
        self.output_dir = output_dir
        self.warhol_dir = warhol_dir
        self.jobname = 'TricolorizeImage'
        self.output_file = output_file

        if not os.path.isdir(output_dir):
            os.mkdir(output_dir)
        self.extra_args = extra_args

        gc3libs.log.info(
            "Tricolorize image %s to %s" % (
                self.grayscaled_image, self.output_file))

        self.tasks = [
            CreateLutApplication(
                self.grayscaled_image,
                "%s.miff" % self.grayscaled_image,
                self.output_dir,
                colors, self.warhol_dir, **extra_args),
            ]

        SequentialTaskCollection.__init__(self, self.tasks)

    def __init__(self, paraFolder, substs, solverParas, **sessionParas):

        logger.debug('entering gParaSearchDriver.__init__')

        self.jobname = sessionParas['jobname'] + 'driver'
        solverParas['jobname'] = self.jobname
        
        self.sessionParas     = sessionParas
        self.solverParas      = solverParas
        self.substs           = substs
 
        self.paraFolder       = paraFolder
        self.iter             = 0

        # create a subfolder for optim over xVar
        self.optimFolder = os.path.join(paraFolder, 'optim' + str(solverParas['xVars'][0]))
        gc3libs.utils.mkdir(self.optimFolder)

        self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas)
        
        self.xVars       = solverParas['xVars']
        self.xParaCombos = solverParas['xInitialParaCombo']
        self.targetVars   = solverParas['targetVars']
        self.target_fx   = solverParas['target_fx']

        self.evaluator = idRiskParaSearchParallel(self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas , **sessionParas)

        initial_task = self.evaluator

        SequentialTaskCollection.__init__(self, self.jobname, [initial_task])

        logger.debug('done solveParaCombination.__init__')

 def __init__(self, name, experiment_id, verbosity,
              submission_id, user_name, parent_id, description=None,
              waiting_time=0):
     '''
     Parameters
     ----------
     name: str
         name of the stage
     experiment_id: int
         ID of the processed experiment
     verbosity: int
         logging verbosity index
     submission_id: int
         ID of the corresponding submission
     user_name: str
         name of the submitting user
     parent_id: int
         ID of the parent
         :class:`Workflow <tmlib.workflow.workflow.Workflow>`
     description: tmlib.tmaps.description.WorkflowStageDescription, optional
         description of the stage (default: ``None``)
     waiting_time: int, optional
         time in seconds that should be waited upon transition from one
         stage to the other to avoid issues related to network file systems
         (default: ``0``)
     '''
     SequentialTaskCollection.__init__(
         self, tasks=None, jobname='%s' % name
     )
     WorkflowStage.__init__(
         self, name=name, experiment_id=experiment_id, verbosity=verbosity,
         submission_id=submission_id, description=description,
         user_name=user_name, parent_id=parent_id
     )
     self.waiting_time = waiting_time

    def __init__(self, init_value, add_value, iterations, **extra_args):
        """
        Create a new tasks that runs `executable` over a set of values
        (initially given by `initial_values_file`, then the output of
        a run is fed as input to the next one), riterating the process
        `total_iterations` times.

        If `slice_size` is a positive integer, then chop the input into
        chunks of -at most- the given size and compute them as separate
        independent jobs.

        Extra keyword arguments are saved and passed down to construct
        the `ValueFunctionIterationApplication`.
        """

        self.init = init_value
        self.increment = add_value
        self.limit = iterations
        self.jobname = "Gdemo_Iteration"

        gc3libs.log.info("Calling DemoIteration.__init__() ... ")

        # create initial task and register it
        initial_task = GdemoApplication(self.init, self.increment, 0)
        SequentialTaskCollection.__init__(self, [initial_task], **extra_args)

    def __init__(self, paraFolder, substs, solverParas, **sessionParas):

        logger.debug("entering gParaSearchDriver.__init__")

        self.jobname = sessionParas["jobname"] + "driver"
        solverParas["jobname"] = self.jobname

        self.sessionParas = sessionParas
        self.solverParas = solverParas
        self.substs = substs

        self.paraFolder = paraFolder
        self.iter = 0

        # create a subfolder for optim over xVar
        self.optimFolder = os.path.join(paraFolder, "optim" + str(solverParas["xVars"][0]))
        self.solverParas["optimFolder"] = self.optimFolder
        gc3libs.utils.mkdir(self.optimFolder)

        self.costlyOptimizer = costlyOptimization.costlyOptimization(solverParas)

        self.xVars = solverParas["xVars"]
        self.xParaCombos = solverParas["xInitialParaCombo"]
        self.targetVars = solverParas["targetVars"]
        self.target_fx = solverParas["target_fx"]

        self.evaluator = idRiskParaSearchParallel(
            self.xVars, self.xParaCombos, substs, self.optimFolder, solverParas, **sessionParas
        )

        initial_task = self.evaluator

        SequentialTaskCollection.__init__(self, self.jobname, [initial_task])

        logger.debug("done solveParaCombination.__init__")

    def __init__(self, joke, **kwargs):

        self.joke = joke
        gc3libs.log.info("\t\t\t\tCalling InnerSequentialFlow.__init__ for joke: {}".format(self.joke))

        self.job_name = joke
        initial_task = B(joke)
        SequentialTaskCollection.__init__(self, [initial_task], **kwargs)

        def __init__(self, name, ABC_UML_IMAGE, output, g3cfile, dimensions, fortran_pes, inputfilelist_abc, **extra_args):

                self.inputfilelist_abc = inputfilelist_abc
                self.output_folder = output
                self.extra_args = extra_args
                self.name = name
# first step --> __init__ source compiling

                first_task = Gfit3C_ABC_uml_Application(abc_uml_image_file=ABC_UML_IMAGE, output_folder=output, g3c_input_file=g3cfile, dimension_file=dimensions, surface_file=fortran_pes, **extra_args)
                SequentialTaskCollection.__init__(self, name, [first_task])

    def __init__(self, param_value, input_file_name, output_folder, iteration, **extra_args):

        gc3libs.log.info("\t\t\t\tCalling InnerSequentialIterationB.__init__ for param [%d] and file [%s]" % (param_value, input_file_name))

        self.param_value = param_value
        self.input_file = input_file_name
        self.output_folder = output_folder
        self.jobname = "Gdemo_InnerSequenceB_"+str(self.param_value)

        initial_task = XtandemApplicationB(param_value, input_file_name, output_folder, iteration)
        SequentialTaskCollection.__init__(self, [initial_task], **extra_args)

    def __init__(self, executable, initial_values_file,
                 total_iterations, slice_size=0,
                 output_dir=TMPDIR, **extra_args):
        """
        Create a new task that runs `executable` over a set of values
        (initially given by `initial_values_file`, then the output of
        a run is fed as input to the next one), iterating the process
        `total_iterations` times.

        If `slice_size` is a positive integer, then chop the input into
        chunks of -at most- the given size and compute them as separate
        independent jobs.

        Extra keyword arguments are saved and passed down to construct
        the `ValueFunctionIterationApplication`.
        """
        assert slice_size >= 0, \
               "Argument `slice_size` to ValueFunctionIteration.__init__" \
               " must be a non-negative integer."

        # remember values for later use
        self.executable = executable
        self.initial_values = initial_values_file
        self.total_iterations = total_iterations - 1
        self.slice_size = slice_size
        self.datadir = output_dir
        self.extra = extra_args

        # count initial values
        if os.path.exists(initial_values_file):
            f = open(initial_values_file, 'r')
            self.num_input_values = gc3libs.utils.count(f, lambda _: True)
            f.close()
        else:
            # XXX: there's no good default value!
            self.num_input_values = 0

        # this little piece of black magic is to ensure intermediate
        # filenames appear numerically sorted in `ls -l` output
        self.values_filename_fmt = ('values.%%0%dd.txt'
                                    % (1 + int(math.log10(total_iterations))))

        self.jobname = extra_args.get('jobname',
                              gc3libs.utils.basename_sans(initial_values_file))

        # create initial task and register it
        initial_task = ValueFunctionIterationPass(executable, initial_values_file,
                                                  0, total_iterations, slice_size,
                                                  self.datadir, self.extra,
                                                  parent=self.jobname)
        SequentialTaskCollection.__init__(self, self.jobname, [initial_task])

 def next(self, x):
     """ensure that the next() is called only once per task."""
     if not hasattr(self, 'next_called_n_times'):
         self.next_called_n_times = 1
     else:
         self.next_called_n_times += 1
     return SequentialTaskCollection.next(self, x)

    def __init__(self, input_bam_folder, **extra_args):
        self.gatkseq = extra_args['gatk_sequence']
        self.S0_output = extra_args['S0_output']
        self.S1_output = extra_args['S1_output']
        self.input_bam_folder = input_bam_folder
        self.name = os.path.basename(input_bam_folder)
        self.extra = extra_args
        self.output_dir = extra_args['output_dir']

        self.jobname = "GATK-" + "_".join(self.gatkseq)
        
        # Get the first stage task in gatk_sequence
        initial_task_fn = getattr(self, "stage%d" % int(self.gatkseq.pop(0)))
        initial_task = initial_task_fn()
        
        SequentialTaskCollection.__init__(self, [initial_task])

    def __init__(self, input_image, copies, ncolors, size=None, **extra_args):
        """XXX do we need input_image and output_image? I guess so?"""
        self.input_image = input_image
        self.output_image = "warhol_%s" % os.path.basename(input_image)
        self.resize = False

        gc3libs.log.info(
            "Producing a warholized version of input file %s " "and store it in %s" % (input_image, self.output_image)
        )

        if size:
            x, y = size.split("x", 2)
            rows = math.sqrt(copies)
            self.resize = "%dx%d" % (int(x) / rows, int(y) / rows)

        self.output_dir = os.path.relpath(extra_args.get("output_dir"))

        self.ncolors = ncolors
        self.copies = copies

        # Check that copies is a perfect square
        if math.sqrt(self.copies) != int(math.sqrt(self.copies)):
            raise gc3libs.exceptions.InvalidArgument("`copies` argument must be a perfect square.")

        self.jobname = extra_args.get("jobname", "WarholizedWorkflow")

        self.grayscaled_image = "grayscaled_%s" % os.path.basename(self.input_image)

        # This is just parsing of the arguments. The last lines, instead,
        # create the initial tasks that will be submitted. By now, we can submit
        # only the first one, `GrayScaleConvertApplication`, which will produce
        # a grayscale image from the input file::
        self.extra_args = extra_args.copy()
        self.extra_args.pop("output_dir")
        self.tasks = [
            GrayScaleConvertApplication(
                self.input_image,
                self.grayscaled_image,
                self.output_dir,
                self.output_dir,
                resize=self.resize,
                **self.extra_args
            )
        ]

        SequentialTaskCollection.__init__(self, self.tasks)

    def __init__(self, **kwargs):

        config = kwargs["config"]
        self.kwargs = kwargs
        gc3libs.log.info(
            "\t Calling MainSequentialFlow.__init({})".format("<No parameters>"))

        self.initial_tasks = []
        if config["file_preparation"]["activated"] == 'True':
            self.initial_tasks = [
                FilePreparation(
                    name="file_preparation",
                    **kwargs)]
        else:
            self.initial_tasks = [SequencewiseParallelFlow(**kwargs)]

        SequentialTaskCollection.__init__(self, self.initial_tasks, **kwargs)

    def __init__(self, substs, solverParas, **sessionParas):

        logger.debug('entering solveParaCombination.__init__ for job %s' % sessionParas['jobname'])
        self.iter    = 0

        self.jobname = 'solverParacombination' + sessionParas['jobname']
        self.substs = substs

        self.sessionParas     = sessionParas
        self.pathToExecutable = sessionParas['pathToExecutable']
        self.architecture     = sessionParas['architecture']
        self.localBaseDir     = sessionParas['localBaseDir']
        
        self.paraFolder = os.path.join(os.getcwd(), sessionParas['jobname'])
        
        self.wBarLower_task = idRiskParaSearchDriver(self.paraFolder, self.substs, solverParas, **sessionParas)

        SequentialTaskCollection.__init__(self, self.jobname, [self.wBarLower_task])

        logger.debug('done gParaSearchDriver.__init__ for job %s' % sessionParas['jobname'])