Python PreprocessedData.create方法代码示例

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
def _insert_preprocessed_data(study, params, prep_template, slq_out,
                              **kwargs):
    """Inserts the preprocessed data to the database

    Parameters
    ----------
    study : Study
        The study to preprocess
    params : BaseParameters
        The parameters to use for preprocessing
    prep_template : PrepTemplate
        The prep template to use for the preprocessing
    slq_out : str
        Path to the split_libraries_fastq.py output directory
    kwargs: ignored
        Necessary to include to support execution via moi.

    Raises
    ------
    ValueError
        If the preprocessed output directory does not contain all the expected
        files
    """
    from os.path import exists, join
    from functools import partial
    from qiita_db.data import PreprocessedData

    # The filepaths that we are interested in are:
    #   1) seqs.fna -> demultiplexed fasta file
    #   2) seqs.fastq -> demultiplexed fastq file
    #   3) seqs.demux -> demultiplexed HDF5 file

    path_builder = partial(join, slq_out)
    fasta_fp = path_builder('seqs.fna')
    fastq_fp = path_builder('seqs.fastq')
    demux_fp = path_builder('seqs.demux')
    log_fp = path_builder('split_library_log.txt')

    # Check that all the files exist
    if not (exists(fasta_fp) and exists(demux_fp) and exists(log_fp)):
        raise ValueError("The output directory %s does not contain all the "
                         "expected files." % slq_out)

    filepaths = [(fasta_fp, "preprocessed_fasta"),
                 (demux_fp, "preprocessed_demux"),
                 (log_fp, "log")]

    if exists(fastq_fp):
        filepaths.append((fastq_fp, "preprocessed_fastq"))

    PreprocessedData.create(study, params._table, params.id, filepaths,
                            prep_template)

    # Change the prep_template status to success
    prep_template.preprocessing_status = 'success'

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
 def test_create_error_data_type(self):
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study,
                                 "preprocessed_sequence_illumina_params",
                                 self.params_id, self.filepaths,
                                 data_type="Metabolomics")
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study,
                                 "preprocessed_sequence_illumina_params",
                                 self.params_id, self.filepaths,
                                 data_type="Metabolomics",
                                 prep_template=self.prep_template)

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
 def test_processing_status_setter_valueerror(self):
     """Raises an error if the processing status is not recognized"""
     ppd = PreprocessedData.create(self.study, self.params_table,
                                   self.params_id, self.filepaths,
                                   data_type="18S")
     with self.assertRaises(ValueError):
         ppd.processing_status = 'not a valid state'

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_create_data_type_only(self):
        # Check that the returned object has the correct id
        obs = PreprocessedData.create(self.study, self.params_table,
                                      self.params_id, self.filepaths,
                                      data_type="18S")
        self.assertEqual(obs.id, 3)

        # Check that the preprocessed data have been correctly added to the DB
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.preprocessed_data WHERE "
            "preprocessed_data_id=3")
        # preprocessed_data_id, preprocessed_params_table,
        # preprocessed_params_id, submitted_to_insdc_status,
        # ebi_submission_accession, ebi_study_accession, data_type_id,
        # link_filepaths_status, vamps_status, processing_status
        exp = [[3, "preprocessed_sequence_illumina_params", 1,
                'not submitted', None, None, 2, 'idle', 'not submitted',
                'not_processed']]
        self.assertEqual(obs, exp)

        # Check that the preprocessed data has been linked with its study
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.study_preprocessed_data WHERE "
            "preprocessed_data_id=3")
        exp = [[1, 3]]
        self.assertEqual(obs, exp)

        # Check that the files have been copied to right location
        exp_fna_fp = join(self.db_test_ppd_dir,
                          "3_%s" % basename(self.fna_fp))
        self.assertTrue(exists(exp_fna_fp))
        self._clean_up_files.append(exp_fna_fp)

        exp_qual_fp = join(self.db_test_ppd_dir,
                           "3_%s" % basename(self.qual_fp))
        self.assertTrue(exists(exp_qual_fp))
        self._clean_up_files.append(exp_qual_fp)

        # Check that the filepaths have been correctly added to the DB
        obs_id = self.conn_handler.execute_fetchone(
            "SELECT count(1) from qiita.filepath")[0]
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.filepath WHERE filepath_id=%d or "
            "filepath_id=%d" % (obs_id - 1, obs_id))
        exp_fna_fp = "3_%s" % basename(self.fna_fp)
        exp_qual_fp = "3_%s" % basename(self.qual_fp)
        # filepath_id, path, filepath_type_id
        exp = [[obs_id - 1, exp_fna_fp, 4, '852952723', 1, 3],
               [obs_id, exp_qual_fp, 5, '852952723', 1, 3]]
        self.assertEqual(obs, exp)

        # Check that the preprocessed data have been correctly
        # linked with the filepaths
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.preprocessed_filepath WHERE "
            "preprocessed_data_id=3")
        # preprocessed_data_id, filepath_id
        self.assertEqual(obs, [[3, obs_id - 1], [3, obs_id]])

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_set_ebi_submission_accession(self):
        new = PreprocessedData.create(
            self.study, self.params_table, self.params_id, self.filepaths,
            raw_data=self.raw_data,
            ebi_submission_accession=self.ebi_submission_accession,
            ebi_study_accession=self.ebi_study_accession)

        new.ebi_submission_accession = 'EBI12345-CC'
        self.assertEqual(new.ebi_submission_accession, 'EBI12345-CC')

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_ebi_study_accession(self):
        new = PreprocessedData.create(
            self.study, self.params_table,
            self.params_id, self.filepaths, prep_template=self.prep_template,
            ebi_submission_accession=self.ebi_submission_accession,
            ebi_study_accession=self.ebi_study_accession)

        new.ebi_study_accession = 'EBI12345-DD'
        self.assertEqual(new.ebi_study_accession, 'EBI12345-DD')

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_processing_status(self):
        """processing_status works correctly"""
        # Processed case
        ppd = PreprocessedData(1)
        self.assertEqual(ppd.processing_status, 'not_processed')

        # not processed case
        ppd = PreprocessedData.create(self.study, self.params_table,
                                      self.params_id, self.filepaths,
                                      data_type="18S")
        self.assertEqual(ppd.processing_status, 'not_processed')

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
 def test_create_error(self):
     """Raises an error if the preprocessed_params_table does not exist"""
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study, "foo", self.params_id,
                                 self.filepaths)
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study, "preprocessed_foo",
                                 self.params_id, self.filepaths)
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study, "foo_params", self.params_id,
                                 self.filepaths)
     with self.assertRaises(IncompetentQiitaDeveloperError):
         PreprocessedData.create(self.study, "preprocessed_foo_params",
                                 self.params_id, self.filepaths)

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
 def test_processing_status_setter(self):
     """Able to update the processing status"""
     ppd = PreprocessedData.create(self.study, self.params_table,
                                   self.params_id, self.filepaths,
                                   data_type="18S")
     self.assertEqual(ppd.processing_status, 'not_processed')
     ppd.processing_status = 'processing'
     self.assertEqual(ppd.processing_status, 'processing')
     ppd.processing_status = 'processed'
     self.assertEqual(ppd.processing_status, 'processed')
     state = 'failed: some error message'
     ppd.processing_status = state
     self.assertEqual(ppd.processing_status, state)

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_create(self):
        """Correctly creates all the rows in the DB for preprocessed data"""
        # Check that the returned object has the correct id
        obs = PreprocessedData.create(
            self.study, self.params_table,
            self.params_id, self.filepaths, prep_template=self.prep_template,
            ebi_submission_accession=self.ebi_submission_accession,
            ebi_study_accession=self.ebi_study_accession)
        self.assertEqual(obs.id, 3)

        # Check that the preprocessed data have been correctly added to the DB
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.preprocessed_data WHERE "
            "preprocessed_data_id=3")
        # preprocessed_data_id, preprocessed_params_table,
        # preprocessed_params_id, submitted_to_insdc_status,
        # ebi_submission_accession, ebi_study_accession, data_type_id,
        # link_filepaths_status, vamps_status, processing_status
        exp = [[3, "preprocessed_sequence_illumina_params", 1,
                'not submitted', "EBI123456-A", "EBI123456-B", 2, 'idle',
                'not submitted', 'not_processed']]
        self.assertEqual(obs, exp)

        # Check that the preprocessed data has been linked with its study
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.study_preprocessed_data WHERE "
            "preprocessed_data_id=3")
        exp = [[1, 3]]
        self.assertEqual(obs, exp)

        # Check that the files have been copied to right location
        exp_fna_fp = join(self.db_test_ppd_dir,
                          "3_%s" % basename(self.fna_fp))
        self.assertTrue(exists(exp_fna_fp))
        self._clean_up_files.append(exp_fna_fp)

        exp_qual_fp = join(self.db_test_ppd_dir,
                           "3_%s" % basename(self.qual_fp))
        self.assertTrue(exists(exp_qual_fp))
        self._clean_up_files.append(exp_qual_fp)

        # Check that the filepaths have been correctly added to the DB
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.filepath WHERE filepath_id=17 or "
            "filepath_id=18")
        exp_fna_fp = "3_%s" % basename(self.fna_fp)
        exp_qual_fp = "3_%s" % basename(self.qual_fp)
        # filepath_id, path, filepath_type_id
        exp = [[17, exp_fna_fp, 4, '852952723', 1, 3],
               [18, exp_qual_fp, 5, '852952723', 1, 3]]
        self.assertEqual(obs, exp)

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_create(self):
        """Correctly creates all the rows in the DB for preprocessed data"""
        # Check that the returned object has the correct id
        obs = PreprocessedData.create(self.study, self.params_table,
                                      self.params_id, self.filepaths,
                                      raw_data=self.raw_data)
        self.assertEqual(obs.id, 3)

        # Check that the preprocessed data have been correctly added to the DB
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.preprocessed_data WHERE "
            "preprocessed_data_id=3")
        # preprocessed_data_id, raw_data_id, preprocessed_params_tables,
        # preprocessed_params_id
        exp = [[3, "preprocessed_sequence_illumina_params", 1, False]]
        self.assertEqual(obs, exp)

        # Check that the preprocessed data has been linked with its study
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.study_preprocessed_data WHERE "
            "preprocessed_data_id=3")
        exp = [[1, 3]]
        self.assertEqual(obs, exp)

        # Check that the files have been copied to right location
        exp_fna_fp = join(self.db_test_ppd_dir,
                          "3_%s" % basename(self.fna_fp))
        self.assertTrue(exists(exp_fna_fp))
        self._clean_up_files.append(exp_fna_fp)

        exp_qual_fp = join(self.db_test_ppd_dir,
                           "3_%s" % basename(self.qual_fp))
        self.assertTrue(exists(exp_qual_fp))
        self._clean_up_files.append(exp_qual_fp)

        # Check that the filepaths have been correctly added to the DB
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.filepath WHERE filepath_id=10 or "
            "filepath_id=11")
        # filepath_id, path, filepath_type_id
        exp = [[10, exp_fna_fp, 4, '852952723', 1],
               [11, exp_qual_fp, 5, '852952723', 1]]
        self.assertEqual(obs, exp)

        # Check that the preprocessed data have been correctly
        # linked with the filepaths
        obs = self.conn_handler.execute_fetchall(
            "SELECT * FROM qiita.preprocessed_filepath WHERE "
            "preprocessed_data_id=3")
        # preprocessed_data_id, filepath_id
        self.assertEqual(obs, [[3, 10], [3, 11]])

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_delete_advanced(self):
        # testing that we can not remove cause preprocessed data has been
        # submitted to EBI or VAMPS
        ppd = PreprocessedData.create(
            self.study, self.params_table,
            self.params_id, self.filepaths, prep_template=self.prep_template)

        # fails due to VAMPS submission
        ppd.update_vamps_status('success')
        with self.assertRaises(QiitaDBStatusError):
            PreprocessedData.delete(ppd.id)
        ppd.update_vamps_status('failed')

        ppd = PreprocessedData(1)
        with self.assertRaises(QiitaDBStatusError):
            PreprocessedData.delete(ppd.id)

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_status(self):
        ppd = PreprocessedData(1)
        self.assertEqual(ppd.status, 'private')

        # Since the status is inferred from the processed data, change the
        # status of the processed data so we can check how it changes in the
        # preprocessed data
        pd = ProcessedData(1)
        pd.status = 'public'
        self.assertEqual(ppd.status, 'public')

        # Check that new preprocessed data has sandbox as status since no
        # processed data exists for them
        ppd = PreprocessedData.create(self.study, self.params_table,
                                      self.params_id, self.filepaths,
                                      data_type="16S")
        self.assertEqual(ppd.status, 'sandbox')

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def test_insert_processed_data_target_gene(self):
        fd, fna_fp = mkstemp(suffix='_seqs.fna')
        close(fd)
        fd, qual_fp = mkstemp(suffix='_seqs.qual')
        close(fd)
        filepaths = [
            (fna_fp, convert_to_id('preprocessed_fasta', 'filepath_type')),
            (qual_fp, convert_to_id('preprocessed_fastq', 'filepath_type'))]

        preprocessed_data = PreprocessedData.create(
            Study(1), "preprocessed_sequence_illumina_params", 1,
            filepaths, data_type="18S")

        params = ProcessedSortmernaParams(1)
        pick_dir = mkdtemp()
        path_builder = partial(join, pick_dir)
        db_path_builder = partial(join, get_mountpoint('processed_data')[0][1])

        # Create a placeholder for the otu table
        with open(path_builder('otu_table.biom'), 'w') as f:
            f.write('\n')

        # Create a placeholder for the directory
        mkdir(path_builder('sortmerna_picked_otus'))

        # Create the log file
        fd, fp = mkstemp(dir=pick_dir, prefix='log_', suffix='.txt')
        close(fd)
        with open(fp, 'w') as f:
            f.write('\n')

        _insert_processed_data_target_gene(preprocessed_data, params, pick_dir)

        new_id = get_count('qiita.processed_data')

        # Check that the files have been copied
        db_files = [db_path_builder("%s_otu_table.biom" % new_id),
                    db_path_builder("%s_sortmerna_picked_otus" % new_id),
                    db_path_builder("%s_%s" % (new_id, basename(fp)))]
        for fp in db_files:
            self.assertTrue(exists(fp))

        # Check that a new preprocessed data has been created
        self.assertTrue(self.conn_handler.execute_fetchone(
            "SELECT EXISTS(SELECT * FROM qiita.processed_data WHERE "
            "processed_data_id=%s)", (new_id, ))[0])

# 需要导入模块: from qiita_db.data import PreprocessedData [as 别名]
# 或者: from qiita_db.data.PreprocessedData import create [as 别名]
    def write_demux_files(self, prep_template, generate_hdf5=True):
        """Writes a demux test file to avoid duplication of code"""
        fna_fp = join(self.temp_dir, 'seqs.fna')
        demux_fp = join(self.temp_dir, 'demux.seqs')
        if generate_hdf5:
            with open(fna_fp, 'w') as f:
                f.write(FASTA_EXAMPLE)
            with File(demux_fp, "w") as f:
                to_hdf5(fna_fp, f)
        else:
            with open(demux_fp, 'w') as f:
                f.write('')

        ppd = PreprocessedData.create(Study(1),
                                      "preprocessed_sequence_illumina_params",
                                      1, [(demux_fp, 6)], prep_template)
        return ppd