Python util.qiime_system_call函数代码示例

    def silly_function(ui):
        for c_value in ui.series(coloring_values):
            sample_ids = sample_ids_from_metadata_description(open(mapping_fp, 'U'),
                '%s:%s' % (coloring_header_name, c_value))

            _headers, _data = filter_mapping_file(data, headers, sample_ids, True)
            per_color_subject_values = list(set([row[subject_index] for row in _data]))

            fd = open(join(output_path, 'color_by_'+c_value+'.txt'), 'w')
            for s in ui.series(per_color_subject_values):
                fd.write('%s\n' % s)
            fd.close()

            if not suppress_trajectory_files:
                for s in ui.series(per_color_subject_values):
                    filename = join(output_path, s+'.txt')

                    if opts.verbose:
                        print 'Working on printing', filename

                    COMMAND_CALL = FILTER_CMD % (coords_fp, mapping_fp,
                        '%s:%s' % (subject_header_name, s), filename,
                        sorting_category)
                    o, e, r = qiime_system_call(COMMAND_CALL)
                    if opts.verbose and e:
                        print 'Error happened on filtering step: \n%s' % e
                        continue

                    COMMAND_CALL = CONVERSION_CMD % (filename, filename)
                    o, e, r = qiime_system_call(COMMAND_CALL)
                    if opts.verbose and e:
                        print 'Error happened on conversion step: \n%s' % e
                        continue # useless here but just in case

def main():
    option_parser, opts, args =\
        parse_command_line_parameters(**script_info)

    if opts.submit_jobs and not opts.make_jobs:
        option_parser.error('Must pass -m if passing -s. (Sorry about this, '
                            'it\'s for backwards-compatibility.)')

    min_args = 2
    if len(args) != min_args:
        option_parser.error('Program requires <commands file> and '
                            '<job prefix>')

    if (len(args[1]) > 10 or len(args[1]) == 0):
        option_parser.error('job prefix must be 1-10 characters long')

    if(not exists(opts.job_dir)):
        try:
            makedirs(opts.job_dir)
        except OSError:
            exit(" Jobs directory can not be created. "
                 "Check for permissions or file with the same name: %s\n"
                 % opts.job_dir)

    commands = list(open(args[0]))
    job_prefix = args[1]

    if opts.mem_per_cpu:
        mem_per_cpu = " --mem_per_cpu=" + opts.mem_per_cpu
    else:
        mem_per_cpu = ""

    if opts.queue:
        queue = " -p " + opts.queue
    else:
        queue = ""

    if (opts.make_jobs):
        filenames = make_jobs(
            commands,
            job_prefix,
            opts.queue,
            opts.job_dir)
    else:
        exit("Should we ever get here???")
    if (opts.submit_jobs):
        for f in filenames:
            qiime_system_call("".join([
                    "sbatch",
                    queue,
                    " -J ", job_prefix,
                    mem_per_cpu,
                    " -o ", normpath(opts.job_dir), sep, job_prefix, "_%j.out",
                    " ", f
                ]), shell=True)

    def test_mothur_supported_version(self):
        """mothur is in path and version is supported """
        acceptable_version = (1, 25, 0)
        self.assertTrue(
            which("mothur"),
            "mothur not found. This may or may not be a problem depending on "
            + "which components of QIIME you plan to use.",
        )
        # mothur creates a log file in cwd, so create a tmp and cd there first
        log_file = join(get_qiime_temp_dir(), "mothur.log")
        command = "mothur \"#set.logfile(name=%s)\" | grep '^mothur v'" % log_file
        stdout, stderr, exit_Status = qiime_system_call(command)

        # remove log file
        remove_files([log_file], error_on_missing=False)

        version_string = stdout.strip().split(" ")[1].strip("v.")
        try:
            version = tuple(map(int, version_string.split(".")))
            pass_test = version == acceptable_version
        except ValueError:
            pass_test = False
            version_string = stdout
        self.assertTrue(
            pass_test,
            "Unsupported mothur version. %s is required, but running %s."
            % (".".join(map(str, acceptable_version)), version_string),
        )

def copy_support_files(file_path):
    """Copy the support files to a named destination 

    file_path: path where you want the support files to be copied to

    Will raise EmperorSupportFilesError if a problem is found whilst trying to
    copy the files.
    """
    file_path = join(file_path, "emperor_required_resources")

    if exists(file_path) == False:
        create_dir(file_path, False)

    # shutil.copytree does not provide an easy way to copy the contents of a
    # directory into another existing directory, hence the system call.
    # use double quotes for the paths to escape any invalid chracter(s)/spaces
    cmd = 'cp -R "%s/"* "%s"' % (get_emperor_support_files_dir(), abspath(file_path))
    cmd_o, cmd_e, cmd_r = qiime_system_call(cmd)

    if cmd_e:
        raise EmperorSupportFilesError, "Error found whilst trying to copy " + "the support files:\n%s\n Could not execute: %s" % (
            cmd_e,
            cmd,
        )

    return

def _iter_ids_over_system_call(cmd_fmt, sample_ids, opts):
    """Iteratively execute a system call over sample IDs

    Parameters
    ----------
    cmd_fmt : str
        The format of the command to execute. It is expected that there
        is a single string format to be done and it should take a sample
        ID
    sample_ids : Iterable of str
        A list of sample IDs of interest

    Returns
    -------
    dict
        A dict containing each sample ID and any errors observed or None if
        no error was observed for the sample. {str: str or None}
    """
    results = {}

    for id_ in sample_ids:
        cmd = cmd_fmt % {'result_path': _result_path(opts, id_),
                         'id': id_}
        stdout, stderr, return_value = qiime_system_call(cmd)

        if return_value != 0:
            msg = stderr.splitlines()
            results[id_] = 'FAILED (%s): %s' % (msg[-1] if msg else '', cmd)
        else:
            results[id_] = None

    return results

def call_commands_serially(commands,
                           status_update_callback,
                           logger,
                           close_logger_on_success=True):
    """Run list of commands, one after another """
    logger.write("Executing commands.\n\n")
    for c in commands:
        for e in c:
            status_update_callback('%s\n%s' % e)
            logger.write('# %s command \n%s\n\n' % e)
            stdout, stderr, return_value = qiime_system_call(e[1])
            if return_value != 0:
                msg = "\n\n*** ERROR RAISED DURING STEP: %s\n" % e[0] +\
                 "Command run was:\n %s\n" % e[1] +\
                 "Command returned exit status: %d\n" % return_value +\
                 "Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
                logger.write(msg)
                logger.close()
                raise WorkflowError, msg
            # in the no error case, we write commands' output to the log
            # and also echo to this proc's stdout/stderr
            else:
                # write stdout and stderr to log file
                logger.write("Stdout:\n%s\nStderr:\n%s\n" % (stdout,stderr))
                # write stdout to stdout
                if stdout:
                    print stdout
                # write stderr to stderr
                if stderr:
                    sys.stderr.write(stderr)
    if close_logger_on_success: logger.close()

def generate_random_password(min_len=8, max_len=12):
    """Returns a random alphanumeric password of random length.

    Returns both unencrypted and encrypted password. Encryption is performed
    via Apache's htpasswd command, using their custom MD5 algorithm.

    Length will be randomly chosen from within the specified bounds
    (inclusive).
    """
    # Modified from
    # http://code.activestate.com/recipes/59873-random-password-generation
    chars = letters + digits
    length = randint(min_len, max_len)
    password = ''.join([choice(chars) for i in range(length)])

    # This is hackish but should work for now...
    stdout, stderr, ret_val = qiime_system_call('htpasswd -nbm foobarbaz %s' %
                                                password)
    if ret_val != 0:
        raise ValueError("Error executing htpasswd command. Do you have this "
                         "command on your machine?")

    # Will be in the form foobarbaz:<encrypted password>
    encrypted_password = stdout.strip().split('foobarbaz:', 1)[1]

    return password, encrypted_password

def run_command(cmd):
    stdout, stderr, ret_val = qiime_system_call(cmd)

    if ret_val != 0:
        raise ExternalCommandFailedError("The command '%s' failed with exit "
                                         "status %d.\n\nStdout:\n\n%s\n\n"
                                         "Stderr:\n\n%s\n" % (cmd,
                                         ret_val, stdout, stderr))

def make_line_plot(
        dir_path, data_file_link, background_color, label_color, xy_coords,
        props, x_len=8, y_len=4, draw_axes=False, generate_eps=True):
    """ Write a line plot

    xy_coords: a dict of form
       {series_label:([x data], [y data], point_marker, color)}

    (code adapted from Micah Hamady's code)
    """
    rc('font', size='8')
    rc('axes', linewidth=.5, edgecolor=label_color)
    rc('axes', labelsize=8)
    rc('xtick', labelsize=8)
    rc('ytick', labelsize=8)
    fig, ax = plt.subplots(figsize=(x_len, y_len))
    mtitle = props.get("title", "Groups")
    x_label = props.get("xlabel", "X")
    y_label = props.get("ylabel", "Y")

    ax.set_title('%s' % mtitle, fontsize='10', color=label_color)
    ax.set_xlabel(x_label, fontsize='8', color=label_color)
    ax.set_ylabel(y_label, fontsize='8', color=label_color)

    sorted_keys = sorted(xy_coords.keys())

    for s_label in sorted_keys:
        s_data = xy_coords[s_label]
        c = s_data[3]
        m = s_data[2]
        ax.plot(s_data[0], s_data[1], c=c, marker=m, label=s_label,
                linewidth=.1, ms=5, alpha=1.0)

    fp = FontProperties()
    fp.set_size('8')
    ax.legend(prop=fp, loc=0)

    img_name = 'scree_plot.png'
    fig.savefig(
        os.path.join(dir_path,
                     img_name),
        dpi=80,
        facecolor=background_color)

    # Create zipped eps files
    eps_link = ""
    if generate_eps:
        eps_img_name = str('scree_plot.eps')
        fig.savefig(os.path.join(dir_path, eps_img_name), format='eps')
        out, err, retcode = qiime_system_call(
            "gzip -f " + os.path.join(dir_path, eps_img_name))
        eps_link = DOWNLOAD_LINK % ((os.path.join(data_file_link,
                                                  eps_img_name) +
                                     ".gz"), "Download Figure")

    return os.path.join(data_file_link, img_name), eps_link

def get_emperor_library_version():
    """Get Emperor version and the git SHA + current branch (if applicable)"""
    emperor_dir = get_emperor_project_dir()
    emperor_version = emperor_library_version

    # more information could be retrieved following this pattern
    sha_cmd = "git --git-dir %s/.git rev-parse HEAD" % (emperor_dir)
    sha_o, sha_e, sha_r = qiime_system_call(sha_cmd)
    git_sha = sha_o.strip()

    branch_cmd = "git --git-dir %s/.git rev-parse --abbrev-ref HEAD" % (emperor_dir)
    branch_o, branch_e, branch_r = qiime_system_call(branch_cmd)
    git_branch = branch_o.strip()

    # validate the output from both command calls
    if is_valid_git_refname(git_branch) and is_valid_git_sha1(git_sha):
        return "%s, %[email protected]%s" % (emperor_version, git_branch, git_sha[0:7])
    else:
        return "%s" % emperor_version

def run_commands(output_dir,commands,run_id,submit_jobs,keep_temp,queue_name):
    """
    """
    job_fps, paths_to_remove = write_job_files(output_dir,commands,run_id,queue_name)
    
    # Call the jobs
    if submit_jobs:
        for job_fp in job_fps:
            qiime_system_call(' '.join(['qsub', job_fp]))
    
    # clean up the shell scripts that were created
    if not keep_temp:
        for p in paths_to_remove:
            try:
                # p is file
                remove(p)
            except OSError:
                # p is directory
                rmtree(p)
    return

def call_cmd(cmd, HALT_EXEC):
    if HALT_EXEC:
        print cmd
        exit(0)
    else:
        stdout, stderr, exit_status = qiime_system_call(cmd)
        if exit_status != 0:
            print "indexdb_rna failed!\nSTDOUT\n%s\nSTDERR\n%s\n" \
                   % (stdout, stderr)
            exit(1)
    return cmd

def submit_jobs(path_to_cluster_jobs, jobs_fp, job_prefix):
    """ Submit the jobs to the queue using cluster_jobs.py
    """
    cmd = '%s -ms %s %s' % (path_to_cluster_jobs, jobs_fp, job_prefix)
    stdout, stderr, return_value = qiime_system_call(cmd)
    if return_value != 0:
        msg = "\n\n*** Could not start parallel jobs. \n" +\
         "Command run was:\n %s\n" % cmd +\
         "Command returned exit status: %d\n" % return_value +\
         "Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
        raise RuntimeError, msg

    def _submit_jobs(self, jobs_fp, job_prefix):
        """ Submit the jobs to the queue using cluster_jobs.py
        """
        cmd = "%s -ms %s %s" % (self._cluster_jobs_fp, jobs_fp, job_prefix)
        stdout, stderr, return_value = qiime_system_call(cmd)
        if return_value != 0:
            msg = (
                "\n\n*** Could not start parallel jobs. \n"
                + "Command run was:\n %s\n" % cmd
                + "Command returned exit status: %d\n" % return_value
                + "Stdout:\n%s\nStderr\n%s\n" % (stdout, stderr)
            )
            raise RuntimeError, msg

        # Leave this comments in as they're useful for debugging.
        # print 'Return value: %d\n' % return_value
        # print 'STDOUT: %s\n' % stdout
        # print 'STDERR: %s\n' % stderr

        return stdout, stderr, return_value

def get_cluster_ratio(fasta_seqs, min_difference_in_clusters):
    """
    Uses uclust to calculate cluster ratio
    cluster_ratio =
    num_of_seq_in_cluster_with_max_seq
    divided by
    num_of_seq_in cluster_with_second_higest_seq
    Parameters
    ----------
    fasta_seqs: list
        list of fasta sequences
    min_difference_in_clusters: float
        percent identity threshold for cluster formation
    Returns
    ----------
    cluster_ratio: float
        cluster ratio of the sequences using uclust
        cluster_ratio =
        num_of_seq_in_cluster_with_max_seq /
        num_of_seq_in cluster_with_second_higest_seq
    """
    cluster_percent_id = min_difference_in_clusters
    temp_dir = get_qiime_temp_dir()
    fd_uc, uclust_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
    close(fd_uc)
    fd_fas, fasta_tempfile_name = mkstemp(dir=temp_dir, suffix='.uc')
    close(fd_fas)
    with open(fasta_tempfile_name, 'w') as fasta_tempfile:
        fasta_tempfile.write(fasta_seqs)
    fasta_tempfile.close()
    count = 0
    command = "uclust --usersort --input {} --uc {} --id 0.98".format(
        fasta_tempfile_name, uclust_tempfile_name)
    # In the function, I am calling uclust a large number of times.
    # Initially I was using from bfillings.get_clusters_from_fasta_filepath
    # but due to issue (biocore/bfillingss#31), I have temporarily
    # reverted to qiime_system_call.

    count_lookup = defaultdict(int)

    qiime_system_call(command)
    uclust_tempfile = open(uclust_tempfile_name, 'r')
    for line in uclust_tempfile:
        if search(r'^C', line):
            pieces = line.split('\t')
            count_lookup[pieces[1]] += int(pieces[2])
            count += 1
    uclust_tempfile.close()
    files_to_be_removed = list()
    files_to_be_removed.append(uclust_tempfile_name)
    remove_files(files_to_be_removed)

    sorted_counts_in_clusters = sorted(
        count_lookup.iteritems(),
        key=lambda x: x[1], reverse=True)
    try:
        max_cluster_count = \
            float(str(sorted_counts_in_clusters[0][1]))
        second_cluster_count = \
            float(str(sorted_counts_in_clusters[1][1]))
        return max_cluster_count / second_cluster_count
    except IndexError:
        return 1