Python job.Job类代码示例

    def post(self, analysis_id):
        command_args = self.get_arguments("commands")
        split = [x.split("#") for x in command_args]
        analysis = Analysis(analysis_id)

        commands = []
        # HARD CODED HACKY THING FOR DEMO, FIX  Issue #164
        fp, mapping_file = mkstemp(suffix="_map_file.txt")
        close(fp)
        SampleTemplate(1).to_file(mapping_file)
        study_fps = {}
        for pd in Study(1).processed_data:
            processed = ProcessedData(pd)
            study_fps[processed.data_type] = processed.get_filepaths()[0][0]
        for data_type, command in split:
            opts = {
                "--otu_table_fp": study_fps[data_type],
                "--mapping_fp": mapping_file
            }
            if command == "Beta Diversity" and data_type in {'16S', '18S'}:
                opts["--tree_fp"] = join(get_db_files_base_dir(), "reference",
                                         "gg_97_otus_4feb2011.tre")
            elif command == "Beta Diversity":
                opts["--parameter_fp"] = join(get_db_files_base_dir(),
                                              "reference", "params_qiime.txt")
            Job.create(data_type, command, opts, analysis)
            commands.append("%s: %s" % (data_type, command))
        user = self.get_current_user()
        self.render("analysis_waiting.html", user=user,
                    aid=analysis_id, aname=analysis.name,
                    commands=commands)
        # fire off analysis run here
        # currently synch run so redirect done here. Will remove after demo
        run_analysis(user, analysis)

def _build_analysis_files(analysis, r_depth=None, **kwargs):
    """Creates the biom tables and mapping file, then adds to jobs

    Parameters
    ----------
    analysis : Analysis object
        The analysis to build files for
    r_depth : int, optional
        Rarefaction depth for biom table creation. Default None
    """
    # create the biom tables and add jobs to the analysis
    analysis.status = "running"
    analysis.build_files(r_depth)
    mapping_file = analysis.mapping_file
    biom_tables = analysis.biom_tables

    # add files to existing jobs
    for job_id in analysis.jobs:
        job = Job(job_id)
        if job.status == 'queued':
            opts = {
                "--otu_table_fp": biom_tables[job.datatype],
                "--mapping_fp": mapping_file
            }
            job_opts = job.options
            job_opts.update(opts)
            job.options = job_opts

    def test_create(self):
        """Makes sure creation works as expected"""
        # make first job
        new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
        self.assertEqual(new.id, 4)
        # make sure job inserted correctly
        obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
                                                 "WHERE job_id = 4")
        exp = [[4, 2, 1, 3, '{"opt1":4}', None]]
        self.assertEqual(obs, exp)
        # make sure job added to analysis correctly
        obs = self.conn_handler.execute_fetchall("SELECT * FROM "
                                                 "qiita.analysis_job WHERE "
                                                 "job_id = 4")
        exp = [[1, 4]]
        self.assertEqual(obs, exp)

        # make second job with diff datatype and command to test column insert
        new = Job.create("16S", "Beta Diversity", {"opt1": 4}, Analysis(1))
        self.assertEqual(new.id, 5)
        # make sure job inserted correctly
        obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
                                                 "WHERE job_id = 5")
        exp = [[5, 1, 1, 2, '{"opt1":4}', None]]
        self.assertEqual(obs, exp)
        # make sure job added to analysis correctly
        obs = self.conn_handler.execute_fetchall("SELECT * FROM "
                                                 "qiita.analysis_job WHERE "
                                                 "job_id = 5")
        exp = [[1, 5]]
        self.assertEqual(obs, exp)

    def test_delete_files(self):
        try:
            Job.delete(1)
            with self.assertRaises(QiitaDBUnknownIDError):
                Job(1)

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.filepath WHERE filepath_id = 12 OR "
                "filepath_id = 19")
            self.assertEqual(obs, [])

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.job_results_filepath WHERE job_id = 1")
            self.assertEqual(obs, [])

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.analysis_job WHERE job_id = 1")
            self.assertEqual(obs, [])

            self.assertFalse(exists(join(self._job_folder,
                             "1_job_result.txt")))
        finally:
            f = join(self._job_folder, "1_job_result.txt")
            if not exists(f):
                with open(f, 'w') as f:
                    f.write("job1result.txt")

    def test_delete_folders(self):
        try:
            Job.delete(2)
            with self.assertRaises(QiitaDBUnknownIDError):
                Job(2)

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.filepath WHERE filepath_id = 13")
            self.assertEqual(obs, [])

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.job_results_filepath WHERE job_id = 2")
            self.assertEqual(obs, [])

            obs = self.conn_handler.execute_fetchall(
                "SELECT * FROM qiita.analysis_job WHERE job_id = 2")
            self.assertEqual(obs, [])

            self.assertFalse(exists(join(self._job_folder, "2_test_folder")))
        finally:
            # put the test data back
            basedir = self._job_folder
            if not exists(join(basedir, "2_test_folder")):
                mkdir(join(basedir, "2_test_folder"))
                mkdir(join(basedir, "2_test_folder", "subdir"))
                with open(join(basedir, "2_test_folder",
                               "testfile.txt"), 'w') as f:
                    f.write("DATA")
                with open(join(basedir, "2_test_folder",
                               "testres.htm"), 'w') as f:
                    f.write("DATA")
                with open(join(basedir, "2_test_folder",
                               "subdir", "subres.html"), 'w') as f:
                    f.write("DATA")

    def _failure_callback(self, msg=None):
        """Executed if something fails"""
        # set the analysis to errored
        self.analysis.status = 'error'

        if self._update_status is not None:
            self._update_status("Failed")

        # set any jobs to errored if they didn't execute
        for job_id in self.analysis.jobs:
            job = Job(job_id)
            if job.status not in {'error', 'completed'}:
                job.status = 'error'

        LogEntry.create('Runtime', msg, info={'analysis': self.analysis.id})

 def test_set_options(self):
     new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
     new.options = self.options
     self.options['--output_dir'] = join(self._job_folder,
                                         '4_alpha_rarefaction.'
                                         'py_output_dir')
     self.assertEqual(new.options, self.options)

 def test_set_options(self):
     new = Job.create("18S", "Alpha Rarefaction", {"opt1": 4}, Analysis(1))
     new.options = self.options
     self.options['--output_dir'] = join(get_db_files_base_dir(),
                                         'job/4_alpha_rarefaction.'
                                         'py_output_dir')
     self.assertEqual(new.options, self.options)

 def test_exists_noexist_return_jobid(self):
     """tests that non-existant job with bad samples returns false"""
     exists, jid = Job.exists(
         "16S", "Beta Diversity",
         {"--otu_table_fp": 1, "--mapping_fp": 27}, Analysis(1),
         return_existing=True)
     self.assertFalse(exists)
     self.assertEqual(jid, None)

 def test_create_exists_return_existing(self):
     """Makes sure creation doesn't duplicate a job by returning existing"""
     Analysis.create(User("[email protected]"), "new", "desc")
     self.conn_handler.execute(
         "INSERT INTO qiita.analysis_sample (analysis_id, "
         "processed_data_id, sample_id) VALUES (3,1,'SKB8.640193'), "
         "(3,1,'SKD8.640184'), (3,1,'SKB7.640196'), (3,1,'SKM9.640192'),"
         "(3,1,'SKM4.640180')")
     new = Job.create("18S", "Beta Diversity",
                      {"--otu_table_fp": 1, "--mapping_fp": 1},
                      Analysis(3), return_existing=True)
     self.assertEqual(new.id, 2)

 def test_exists(self):
     # need to insert matching sample data into analysis 2
     self.conn_handler.execute(
         "DELETE FROM qiita.analysis_sample WHERE analysis_id = 2")
     self.conn_handler.execute(
         "INSERT INTO qiita.analysis_sample (analysis_id, "
         "processed_data_id, sample_id) VALUES (2,1,'SKB8.640193'), "
         "(2,1,'SKD8.640184'), (2,1,'SKB7.640196'), (2,1,'SKM9.640192'),"
         "(2,1,'SKM4.640180')")
     """tests that existing job returns true"""
     self.assertTrue(Job.exists("18S", "Beta Diversity",
                                {"--otu_table_fp": 1,
                                 "--mapping_fp": 1}, Analysis(1)))

def run_analysis(user, analysis):
    """Run the commands within an Analysis object and sends user messages"""
    analysis.status = "running"
    all_good = True
    pubsub = r_server.pubsub()
    pubsub.subscribe(user)
    for job_id in analysis.jobs:
        job = Job(job_id)
        if job.status == 'queued':
            name, command = job.command
            options = job.options
            # create json base for websocket messages
            msg = {
                "analysis": analysis.id,
                "msg": None,
                "command": "%s: %s" % (job.datatype, name)
            }

            o_fmt = ' '.join(['%s %s' % (k, v) for k, v in options.items()])
            c_fmt = str("%s %s" % (command, o_fmt))

            # send running message to user wait page
            job.status = 'running'
            msg["msg"] = "Running"
            r_server.rpush(user + ":messages", dumps(msg))
            r_server.publish(user, dumps(msg))

            # run the command
            try:
                qiita_compute.submit_sync(c_fmt)
            except Exception as e:
                all_good = False
                job.status = 'error'
                msg["msg"] = "ERROR"
                r_server.rpush(user + ":messages", dumps(msg))
                r_server.publish(user, dumps(msg))
                print("Failed compute on job id %d: %s\n%s" %
                      (job_id, e, c_fmt))
                continue

            msg["msg"] = "Completed"
            r_server.rpush(user + ":messages", dumps(msg))
            r_server.publish(user, dumps(msg))
            # FIX THIS Should not be hard coded
            job.add_results([(options["--output_dir"], "directory")])
            job.status = 'completed'

    # send websockets message that we are done
    msg["msg"] = "allcomplete"
    msg["command"] = ""
    r_server.rpush(user + ":messages", dumps(msg))
    r_server.publish(user, dumps(msg))
    pubsub.unsubscribe()
    # set final analysis status
    if all_good:
        analysis.status = "completed"
    else:
        analysis.status = "error"

 def test_exists_noexist_options(self):
     # need to insert matching sample data into analysis 2
     # makes sure failure is because options and not samples
     self.conn_handler.execute(
         "DELETE FROM qiita.analysis_sample WHERE analysis_id = 2")
     self.conn_handler.execute(
         "INSERT INTO qiita.analysis_sample (analysis_id, "
         "processed_data_id, sample_id) VALUES (2,1,'SKB8.640193'), "
         "(2,1,'SKD8.640184'), (2,1,'SKB7.640196'), (2,1,'SKM9.640192'),"
         "(2,1,'SKM4.640180')")
     """tests that non-existant job with bad options returns false"""
     self.assertFalse(Job.exists("18S", "Beta Diversity",
                                 {"--otu_table_fp": 1,
                                  "--mapping_fp": 27}, Analysis(1)))

def system_call_from_job(job_id, **kwargs):
    """Executes a system call described by a Job

    Parameters
    ----------
    job_id : int
        The job object ID
    """
    job = Job(job_id)
    name, command = job.command
    options = job.options

    cmd = [command]
    cmd.extend(flatten(options.items()))
    cmd_fmt = ' '.join((str(i) for i in cmd))

    try:
        so, se, status = system_call(cmd_fmt)
    except Exception as e:
        job.set_error(str(e))
        raise

    # FIX THIS add_results should not be hard coded  Issue #269
    job.add_results([(job.options["--output_dir"], "directory")])

 def test_exists_return_jobid(self):
     """tests that existing job returns true"""
     # need to insert matching sample data into analysis 2
     self.conn_handler.execute(
         "DELETE FROM qiita.analysis_sample WHERE analysis_id = 2")
     self.conn_handler.execute(
         "INSERT INTO qiita.analysis_sample "
         "(analysis_id, processed_data_id, sample_id) VALUES "
         "(2, 1,'1.SKB8.640193'), (2, 1,'1.SKD8.640184'), "
         "(2, 1,'1.SKB7.640196'), (2, 1,'1.SKM9.640192'), "
         "(2, 1,'1.SKM4.640180')")
     exists, jid = Job.exists("18S", "Beta Diversity",
                              {"--otu_table_fp": 1, "--mapping_fp": 1},
                              Analysis(1), return_existing=True)
     self.assertTrue(exists)
     self.assertEqual(jid, Job(2))