Python parse.parse_qiime_parameters函数代码示例

    def test_run_core_diversity_analyses_error_on_invalid_category(self):
        """run_core_diversity_analyses: error raised on invalid categories
        """
        # too few values in 'month' category
        self.assertRaises(ValueError,
                          run_core_diversity_analyses,
                          self.test_data['biom'][0],
                          self.test_data['map'][0],
                          20,
                          output_dir=self.test_out,
                          params=parse_qiime_parameters({}),
                          qiime_config=self.qiime_config,
                          categories=['SampleType', 'month'],
                          tree_fp=self.test_data['tree'][0],
                          parallel=False,
                          status_update_callback=no_status_updates)

        # invalid category name
        self.assertRaises(ValueError,
                          run_core_diversity_analyses,
                          self.test_data['biom'][0],
                          self.test_data['map'][0],
                          20,
                          output_dir=self.test_out,
                          params=parse_qiime_parameters({}),
                          qiime_config=self.qiime_config,
                          categories=['not-a-real-category'],
                          tree_fp=self.test_data['tree'][0],
                          parallel=False,
                          status_update_callback=no_status_updates)

    def run_command(self, options, arguments):

        verbose = options["verbose"]

        input_fp = options["input_fp"]
        output_dir = options["output_dir"]
        verbose = options["verbose"]
        print_only = options["print_only"]

        parallel = options["parallel"]
        # No longer checking that jobs_to_start > 2, but
        # commenting as we may change our minds about this.
        # if parallel: raise_error_on_parallel_unavailable()

        if options["parameter_fp"]:
            try:
                parameter_f = open(options["parameter_fp"])
            except IOError:
                raise QiimeCommandError, "Can't open parameters file (%s). Does it exist? Do you have read access?" % options[
                    "parameter_fp"
                ]
            params = parse_qiime_parameters(parameter_f)
        else:
            params = parse_qiime_parameters([])
            # empty list returns empty defaultdict for now

        params["parallel"]["jobs_to_start"] = self._validate_jobs_to_start(
            options["jobs_to_start"], qiime_config["jobs_to_start"], parallel
        )

        try:
            makedirs(output_dir)
        except OSError:
            if options["force"]:
                pass
            else:
                # Since the analysis can take quite a while, I put this check
                # in to help users avoid overwriting previous output.
                print "Output directory already exists. Please choose " + "a different directory, or force overwrite with -f."
                exit(1)

        if print_only:
            command_handler = print_commands
        else:
            command_handler = call_commands_serially

        if verbose:
            status_update_callback = print_to_stdout
        else:
            status_update_callback = no_status_updates

        run_qiime_data_preparation(
            input_fp,
            output_dir,
            command_handler=command_handler,
            params=params,
            qiime_config=qiime_config,
            parallel=parallel,
            status_update_callback=status_update_callback,
        )

def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    verbose = opts.verbose

    input_fp = opts.input_fp
    output_dir = opts.output_dir
    verbose = opts.verbose
    print_only = opts.print_only

    parallel = opts.parallel
    # No longer checking that jobs_to_start > 2, but
    # commenting as we may change our minds about this.
    # if parallel: raise_error_on_parallel_unavailable()

    if opts.parameter_fp:
        try:
            parameter_f = open(opts.parameter_fp)
        except IOError:
            raise IOError, "Can't open parameters file (%s). Does it exist? Do you have read access?" % opts.parameter_fp
        params = parse_qiime_parameters(parameter_f)
    else:
        params = parse_qiime_parameters([])
        # empty list returns empty defaultdict for now

    jobs_to_start = opts.jobs_to_start
    default_jobs_to_start = qiime_config["jobs_to_start"]
    validate_and_set_jobs_to_start(params, jobs_to_start, default_jobs_to_start, parallel, option_parser)

    try:
        makedirs(output_dir)
    except OSError:
        if opts.force:
            pass
        else:
            # Since the analysis can take quite a while, I put this check
            # in to help users avoid overwriting previous output.
            print "Output directory already exists. Please choose " + "a different directory, or force overwrite with -f."
            exit(1)

    if print_only:
        command_handler = print_commands
    else:
        command_handler = call_commands_serially

    if verbose:
        status_update_callback = print_to_stdout
    else:
        status_update_callback = no_status_updates

    run_qiime_data_preparation(
        input_fp,
        output_dir,
        command_handler=command_handler,
        params=params,
        qiime_config=qiime_config,
        parallel=parallel,
        status_update_callback=status_update_callback,
    )

def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)
    verbose = opts.verbose
    
    otu_table_fp = opts.otu_table_fp
    output_dir = opts.output_dir
    mapping_fp = opts.mapping_fp
    verbose = opts.verbose
    print_only = opts.print_only
    mapping_cat = opts.mapping_category
    sort=opts.sort
    
    if mapping_cat != None and mapping_fp == None:
        option_parser.error("If passing -c must also pass -m.")
    
    if opts.parameter_fp:
        try:
            parameter_f = open(opts.parameter_fp, 'U')
        except IOError:
            raise IOError,\
                "Can't open parameters file (%s). Does it exist? Do you have read access?" \
                 % opts.parameter_fp
        params = parse_qiime_parameters(parameter_f)
        parameter_f.close()
    else:
        params = parse_qiime_parameters([]) 
        # empty list returns empty defaultdict for now
                                   
    try:
        makedirs(output_dir)
    except OSError:
        if opts.force:
            pass
        else:
            # Since the analysis can take quite a while, I put this check
            # in to help users avoid overwriting previous output.
            option_parser.error("Output directory already exists. Please choose"
                " a different directory, or force overwrite with -f.")

    if print_only:
        command_handler = print_commands
    else:
        command_handler = call_commands_serially
    
    if verbose:
        status_update_callback = print_to_stdout
    else:
        status_update_callback = no_status_updates
     
    run_summarize_taxa_through_plots(
     otu_table_fp=otu_table_fp,
     mapping_fp=mapping_fp,
     output_dir=output_dir,
     mapping_cat=mapping_cat,
     sort=sort,
     command_handler=command_handler,
     params=params,
     qiime_config=qiime_config,
     status_update_callback=status_update_callback)

def main():
    option_parser, opts, args = \
        parse_command_line_parameters(suppress_verbose=True, **script_info)
        
    input_dir = opts.input_dir
    parameter_fp = opts.parameter_fp
    read1_indicator = opts.read1_indicator
    read2_indicator = opts.read2_indicator
    match_barcodes = opts.match_barcodes
    barcode_indicator = opts.barcode_indicator
    leading_text = opts.leading_text
    trailing_text = opts.trailing_text
    include_input_dir_path = opts.include_input_dir_path
    output_dir = abspath(opts.output_dir)
    remove_filepath_in_name = opts.remove_filepath_in_name
    print_only = opts.print_only
    
    if remove_filepath_in_name and not include_input_dir_path:
        option_parser.error("If --remove_filepath_in_name is enabled, "
            "--include_input_dir_path must also be enabled.")

    if opts.parameter_fp:
        with open(opts.parameter_fp, 'U') as parameter_f:
            params_dict = parse_qiime_parameters(parameter_f)
        params_str = get_params_str(params_dict['join_paired_ends'])
    else:
        params_dict = {}
        params_str = ""

    create_dir(output_dir)
    
    all_files = []
    extensions = ['.fastq.gz', '.fastq', '.fq.gz', '.fq']
    
    for root, dir, fps in walk(input_dir):
        for fp in fps:
            for extension in extensions:
                if fp.endswith(extension):
                    all_files += [abspath(join(root, fp))]
        
    pairs, bc_pairs = get_pairs(all_files, read1_indicator, read2_indicator,
        match_barcodes, barcode_indicator)

    commands = create_commands_jpe(pairs, output_dir,
        params_str, leading_text, trailing_text, include_input_dir_path,
        remove_filepath_in_name, match_barcodes, bc_pairs)
        
    qiime_config = load_qiime_config()
    if print_only:
        command_handler = print_commands
    else:
        command_handler = call_commands_serially
    logger = WorkflowLogger(generate_log_fp(output_dir),
                            params=params_dict,
                            qiime_config=qiime_config)
    # Call the command handler on the list of commands
    command_handler(commands,
                    status_update_callback=no_status_updates,
                    logger=logger,
                    close_logger_on_success=True)

    def test_run_core_diversity_analyses_parallel(self):
        """run_core_diversity_analyses functions with categories in parallel
        """
        run_core_diversity_analyses(
            self.test_data['biom'][0],
            self.test_data['map'][0],
            20,
            output_dir=self.test_out,
            params=parse_qiime_parameters({}),
            arare_num_steps=3,
            qiime_config=self.qiime_config,
            categories=['SampleType', 'days_since_epoch'],
            tree_fp=self.test_data['tree'][0],
            parallel=True,
            status_update_callback=no_status_updates)

        # Basic sanity test that output directories and files exist
        fps = [
            '%s/bdiv_even20' % self.test_out,
            '%s/arare_max20' % self.test_out,
            '%s/taxa_plots' % self.test_out,
            '%s/bdiv_even20/unweighted_unifrac_dm.txt' % self.test_out,
            '%s/bdiv_even20/weighted_unifrac_pc.txt' % self.test_out,
            '%s/arare_max20/compare_chao1/days_since_epoch_stats.txt' % self.test_out,
            '%s/arare_max20/compare_PD_whole_tree/SampleType_boxplots.pdf' % self.test_out,
            '%s/index.html' % self.test_out,
            '%s/table_mc%d.biom.gz' % (self.test_out, 20)
        ]
        for fp in fps:
            self.assertTrue(exists(fp))

    def setUp(self):
        """setup the test values"""
        
        self.qiime_config = load_qiime_config()
        self.dirs_to_remove = []
        self.files_to_remove = []
        
        #this is specific to the web-apps only
        test_dir = abspath(dirname(__file__))
        self.fna_original_fp = os.path.join(test_dir, 'support_files', \
                                        'test.fna')

        tmp_dir = self.qiime_config['temp_dir'] or '/tmp/'
        if not exists(tmp_dir):
            makedirs(tmp_dir)
            
            # if test creates the temp dir, also remove it
            self.dirs_to_remove.append(tmp_dir)
        
        self.wf_out = get_tmp_filename(tmp_dir=tmp_dir,
         prefix='qiime_wf_out',suffix='',result_constructor=str)
        if not exists(self.wf_out):
            makedirs(self.wf_out)         
            self.dirs_to_remove.append(self.wf_out)
        #print self.wf_out
        working_dir = self.qiime_config['working_dir'] or './'
        jobs_dir = join(working_dir,'jobs')
        if not exists(jobs_dir):
            # only clean up the jobs dir if it doesn't already exist
            self.dirs_to_remove.append(jobs_dir)
        self.params = parse_qiime_parameters(qiime_parameters_f.split('\n'))

        signal.signal(signal.SIGALRM, timeout)
        # set the 'alarm' to go off in allowed_seconds seconds
        signal.alarm(allowed_seconds_per_test)

    def setUp(self):
        """ """
        self.test_data = get_test_data_fps()
        self.files_to_remove = []
        self.dirs_to_remove = []

        # Create example output directory
        tmp_dir = get_qiime_temp_dir()
        self.test_out = mkdtemp(dir=tmp_dir,
                                prefix='core_qiime_analyses_test_',
                                suffix='')
        self.dirs_to_remove.append(self.test_out)

        self.qiime_config = load_qiime_config()
        self.params = parse_qiime_parameters(params_f1)

        # suppress stderr during tests (one of the systems calls in the
        # workflow prints a warning, and we can't suppress that warning with
        # warnings.filterwarnings) here because it comes from within the code
        # executed through the system call. Found this trick here:
        # http://stackoverflow.com/questions/9949633/suppressing-print-as-stdout-python
        self.saved_stderr = sys.stderr
        sys.stderr = StringIO()

        initiate_timeout(180)

    def test_run_core_diversity_analyses(self):
        """run_core_diversity_analyses functions with categories
        """
        run_core_diversity_analyses(
            self.test_data["biom"][0],
            self.test_data["map"][0],
            20,
            output_dir=self.test_out,
            params=parse_qiime_parameters({}),
            qiime_config=self.qiime_config,
            categories=["SampleType", "days_since_epoch"],
            tree_fp=self.test_data["tree"][0],
            parallel=False,
            status_update_callback=no_status_updates,
        )

        # Basic sanity test that output directories and files exist
        fps = [
            "%s/bdiv_even20" % self.test_out,
            "%s/arare_max20" % self.test_out,
            "%s/taxa_plots" % self.test_out,
            "%s/bdiv_even20/unweighted_unifrac_dm.txt" % self.test_out,
            "%s/bdiv_even20/weighted_unifrac_pc.txt" % self.test_out,
            "%s/arare_max20/compare_chao1/days_since_epoch_stats.txt" % self.test_out,
            "%s/arare_max20/compare_PD_whole_tree/SampleType_boxplots.pdf" % self.test_out,
            "%s/index.html" % self.test_out,
            "%s/table_mc%d.biom.gz" % (self.test_out, 20),
        ]
        for fp in fps:
            self.assertTrue(exists(fp))

    def test_run_core_diversity_analyses_no_categories(self):
        """run_core_diversity_analyses functions without categories
        """
        # this takes a long time, so use a longer sigalrm
        run_core_diversity_analyses(
            self.test_data['biom'][0],
            self.test_data['map'][0],
            20,
            output_dir=self.test_out,
            params=parse_qiime_parameters({}),
            qiime_config=self.qiime_config,
            categories=None,
            tree_fp=self.test_data['tree'][0],
            parallel=False,
            status_update_callback=no_status_updates)

        # Basic sanity test that output directories and files exist
        fps = [
            '%s/bdiv_even20' % self.test_out,
            '%s/arare_max20' % self.test_out,
            '%s/taxa_plots' % self.test_out,
            '%s/bdiv_even20/unweighted_unifrac_dm.txt' % self.test_out,
            '%s/bdiv_even20/weighted_unifrac_pc.txt' % self.test_out,
            '%s/index.html' % self.test_out,
            '%s/table_mc%d.biom.gz' % (self.test_out, 20)
        ]
        for fp in fps:
            self.assertTrue(exists(fp))

        # categorical output files don't exist
        self.assertFalse(exists(
            '%s/arare_max20/compare_chao1/days_since_epoch_stats.txt' % self.test_out))
        self.assertFalse(exists(
            '%s/arare_max20/compare_PD_whole_tree/SampleType_boxplots.pdf' % self.test_out))

    def test_run_core_diversity_analyses_no_tree(self):
        """run_core_diversity_analyses functions without tree
        """
        # this takes a long time, so use a longer sigalrm
        run_core_diversity_analyses(
            self.test_data['biom'][0],
            self.test_data['map'][0],
            20,
            output_dir=self.test_out,
            params=parse_qiime_parameters(
                ['beta_diversity:metrics bray_curtis',
                 'alpha_diversity:metrics observed_species,chao1']),
            qiime_config=self.qiime_config,
            categories=['SampleType'],
            tree_fp=None,
            parallel=False,
            status_update_callback=no_status_updates)

        # Basic sanity test that output directories and files exist
        fps = [
            '%s/bdiv_even20' % self.test_out,
            '%s/arare_max20' % self.test_out,
            '%s/taxa_plots' % self.test_out,
            '%s/bdiv_even20/bray_curtis_dm.txt' % self.test_out,
            '%s/arare_max20/compare_observed_species/SampleType_boxplots.pdf' % self.test_out,
            '%s/index.html' % self.test_out,
            '%s/table_mc%d.biom.gz' % (self.test_out, 20)
        ]
        for fp in fps:
            self.assertTrue(exists(fp))

        # phylogenetic diversity output files do not exist
        self.assertFalse(exists(
            '%s/bdiv_even20/unweighted_unifrac_dm.txt' % self.test_out))

    def setUp(self):
        """ """
        
        self.qiime_config = load_qiime_config()
        self.dirs_to_remove = []
        self.files_to_remove = []
        
        #this is specific to the web-apps only
        test_dir = abspath(dirname(__file__))
        sff_original_fp = os.path.join(test_dir, 'support_files', \
                                        'Fasting_subset.sff')
        
        self.sff_fp = os.path.join('/%s/' % environ['HOME'], 
                                   'Fasting_subset.sff')
        self.files_to_remove.append(self.sff_fp)
        copy(sff_original_fp, self.sff_fp)
        
        self.illumina_fps = [os.path.join(test_dir, 'support_files', \
                                        's_8_1_sequence_100_records.txt'),
                             os.path.join(test_dir, 'support_files', \
                                        's_8_2_sequence_100_records.txt')]
        self.illumina_map_fp = os.path.join(test_dir, 'support_files', \
                                        's8_map_incomplete.txt')
    
        self.fasta_fps=[os.path.join(test_dir,'support_files',
                                   'test_split_lib_seqs.fasta')]
        self.fasta_map_fp = os.path.join(test_dir, 'support_files', \
                                        'fasta_mapping_file.txt')

        tmp_dir = "/%s/test_wf" % environ['HOME']
        self.dirs_to_remove.append(tmp_dir)
        
        #self.qiime_config['temp_dir'] or '/tmp/'
        if not exists(tmp_dir):
            makedirs(tmp_dir)
            # if test creates the temp dir, also remove it
            #self.dirs_to_remove.append(tmp_dir)
            
        self.wf_out="/%s/test_processed_data" % environ['HOME']
        #print self.wf_out
        self.dirs_to_remove.append(self.wf_out)
        self.gg_out=os.path.join(self.wf_out,'gg_97_otus')
        if not exists(self.gg_out):
            makedirs(self.gg_out)
            #self.dirs_to_remove.append(self.gg_out)
            
        self.fasting_mapping_fp = get_tmp_filename(tmp_dir=tmp_dir,
         prefix='qiime_wf_mapping',suffix='.txt')
        fasting_mapping_f = open(self.fasting_mapping_fp,'w')
        fasting_mapping_f.write(fasting_map)
        fasting_mapping_f.close()
        self.files_to_remove.append(self.fasting_mapping_fp)
        
        self.params = parse_qiime_parameters(qiime_parameters_f)

        signal.signal(signal.SIGALRM, timeout)
        # set the 'alarm' to go off in allowed_seconds seconds
        signal.alarm(allowed_seconds_per_test)

    def setUp(self):
        """ """
        self.test_data = get_test_data_fps()
        self.files_to_remove = []
        self.dirs_to_remove = []

        # Create example output directory
        tmp_dir = get_qiime_temp_dir()
        self.test_out = mkdtemp(dir=tmp_dir,
                                prefix='core_qiime_analyses_test_',
                                suffix='')
        self.dirs_to_remove.append(self.test_out)

        self.qiime_config = load_qiime_config()
        self.params = parse_qiime_parameters([])
        self.params_sortmerna = parse_qiime_parameters(
            ['pick_otus:otu_picking_method\tsortmerna'])
        self.params_sumaclust = parse_qiime_parameters(
            ['pick_otus:otu_picking_method\tsumaclust'])

        initiate_timeout(60)

    def setUp(self):
        """ """
        
        self.qiime_config = load_qiime_config()
        self.dirs_to_remove = []
        self.files_to_remove = []
        
        #this is specific to the web-apps only
        test_dir = abspath(dirname(__file__))
        sff_original_fp = os.path.join(test_dir, 'support_files', \
                                        'Fasting_subset.sff')

        # copy sff file to working directory
        self.sff_dir = tempfile.mkdtemp()
        self.dirs_to_remove.append(self.sff_dir)
        
        self.sff_fp = os.path.join(self.sff_dir, 'Fasting_subset.sff')
        copy(sff_original_fp, self.sff_fp)
        self.files_to_remove.append(self.sff_fp)
        
        tmp_dir = self.qiime_config['temp_dir'] or '/tmp/'
        if not exists(tmp_dir):
            makedirs(tmp_dir)
            # if test creates the temp dir, also remove it
            self.dirs_to_remove.append(tmp_dir)
        
        self.wf_out = get_tmp_filename(tmp_dir=tmp_dir,
         prefix='qiime_wf_out',suffix='',result_constructor=str)
        self.dirs_to_remove.append(self.wf_out)
        
        self.fasting_mapping_fp = get_tmp_filename(tmp_dir=tmp_dir,
         prefix='qiime_wf_mapping',suffix='.txt')
        fasting_mapping_f = open(self.fasting_mapping_fp,'w')
        fasting_mapping_f.write(fasting_map)
        
        fasting_mapping_f.close()
        self.files_to_remove.append(self.fasting_mapping_fp)
        
        working_dir = self.qiime_config['working_dir'] or './'
        jobs_dir = join(working_dir,'jobs')
        if not exists(jobs_dir):
            # only clean up the jobs dir if it doesn't already exist
            self.dirs_to_remove.append(jobs_dir)
        self.params = parse_qiime_parameters(qiime_parameters_f.split('\n'))

        signal.signal(signal.SIGALRM, timeout)
        # set the 'alarm' to go off in allowed_seconds seconds
        signal.alarm(allowed_seconds_per_test)

    def setUp(self):
        """ """
        self.test_data = get_test_data_fps()
        self.files_to_remove = []
        self.dirs_to_remove = []

        # Create example output directory
        tmp_dir = get_qiime_temp_dir()
        self.test_out = mkdtemp(dir=tmp_dir,
                                prefix='core_qiime_analyses_test_',
                                suffix='')
        self.dirs_to_remove.append(self.test_out)

        self.qiime_config = load_qiime_config()
        self.params = parse_qiime_parameters([])

        initiate_timeout(60)