Python EBISubmission.generate_demultiplexed_fastq方法代码示例

# 需要导入模块: from qiita_ware.ebi import EBISubmission [as 别名]
# 或者: from qiita_ware.ebi.EBISubmission import generate_demultiplexed_fastq [as 别名]
def submit_EBI(artifact_id, action, send, test=False, test_size=False):
    """Submit an artifact to EBI

    Parameters
    ----------
    artifact_id : int
        The artifact id
    action : %s
        The action to perform with this data
    send : bool
        True to actually send the files
    test : bool
        If True some restrictions will be ignored, only used in parse_EBI_reply
    test_size : bool
        If True the EBI-ENA restriction size will be changed to 6000
    """
    # step 1: init and validate
    ebi_submission = EBISubmission(artifact_id, action)

    # step 2: generate demux fastq files
    try:
        ebi_submission.generate_demultiplexed_fastq()
    except Exception:
        error_msg = format_exc()
        if isdir(ebi_submission.full_ebi_dir):
            rmtree(ebi_submission.full_ebi_dir)
        LogEntry.create('Runtime', error_msg,
                        info={'ebi_submission': artifact_id})
        raise

    # step 3: generate and write xml files
    ebi_submission.generate_xml_files()

    # before we continue let's check the size of the submission
    to_review = [ebi_submission.study_xml_fp,
                 ebi_submission.sample_xml_fp,
                 ebi_submission.experiment_xml_fp,
                 ebi_submission.run_xml_fp,
                 ebi_submission.submission_xml_fp]
    total_size = sum([stat(tr).st_size for tr in to_review if tr is not None])
    # note that the max for EBI is 10M but let's play it safe
    max_size = 8.5e+6 if not test_size else 6000
    if total_size > max_size:
        LogEntry.create(
            'Runtime', 'The submission: %d is larger than allowed (%d), will '
            'try to fix: %d' % (artifact_id, max_size, total_size))
        # transform current metadata to dataframe for easier curation
        rows = {k: dict(v) for k, v in viewitems(ebi_submission.samples)}
        df = pd.DataFrame.from_dict(rows, orient='index')
        # remove unique columns and same value in all columns
        nunique = df.apply(pd.Series.nunique)
        nsamples = len(df.index)
        cols_to_drop = set(
            nunique[(nunique == 1) | (nunique == nsamples)].index)
        # maximize deletion by removing also columns that are almost all the
        # same or almost all unique
        cols_to_drop = set(
            nunique[(nunique <= int(nsamples * .01)) |
                    (nunique >= int(nsamples * .5))].index)
        cols_to_drop = cols_to_drop - {'taxon_id', 'scientific_name',
                                       'description'}
        all_samples = ebi_submission.sample_template.ebi_sample_accessions
        samples = {k: all_samples[k] for k in ebi_submission.samples}
        ebi_submission.write_xml_file(
            ebi_submission.generate_sample_xml(samples, cols_to_drop),
            ebi_submission.sample_xml_fp)

        # now let's recalculate the size to make sure it's fine
        new_total_size = sum([stat(tr).st_size
                              for tr in to_review if tr is not None])
        LogEntry.create(
            'Runtime', 'The submission: %d after cleaning is %d and was %d' % (
                artifact_id, total_size, new_total_size))
        if new_total_size > max_size:
            raise ComputeError(
                'Even after cleaning the submission: %d is too large. Before '
                'cleaning: %d, after: %d' % (
                    artifact_id, total_size, new_total_size))

    if send:
        # getting aspera's password
        old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
        if old_ascp_pass == '':
            environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
        ascp_passwd = environ['ASPERA_SCP_PASS']
        LogEntry.create('Runtime',
                        ('Submission of sequences of pre_processed_id: '
                         '%d completed successfully' % artifact_id))

        # step 4: sending sequences
        if action != 'MODIFY':
            LogEntry.create('Runtime',
                            ("Submitting sequences for pre_processed_id: "
                             "%d" % artifact_id))
            for cmd in ebi_submission.generate_send_sequences_cmd():
                stdout, stderr, rv = system_call(cmd)
                if rv != 0:
                    error_msg = ("ASCP Error:\nStd output:%s\nStd error:%s" % (
                        stdout, stderr))
                    environ['ASPERA_SCP_PASS'] = old_ascp_pass
#.........这里部分代码省略.........

# 需要导入模块: from qiita_ware.ebi import EBISubmission [as 别名]
# 或者: from qiita_ware.ebi.EBISubmission import generate_demultiplexed_fastq [as 别名]
def submit_EBI(preprocessed_data_id, action, send):
    """Submit a preprocessed data to EBI

    Parameters
    ----------
    preprocessed_data_id : int
        The preprocesssed data id
    action : %s
        The action to perform with this data
    send : bool
        True to actually send the files
    """
    # step 1: init and validate
    ebi_submission = EBISubmission(preprocessed_data_id, action)

    # step 2: generate demux fastq files
    ebi_submission.study.ebi_submission_status = 'submitting'
    try:
        ebi_submission.generate_demultiplexed_fastq()
    except:
        error_msg = format_exc()
        if isdir(ebi_submission.full_ebi_dir):
            rmtree(ebi_submission.full_ebi_dir)
        ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
        LogEntry.create('Runtime', error_msg,
                        info={'ebi_submission': preprocessed_data_id})
        raise

    # step 3: generate and write xml files
    ebi_submission.generate_xml_files()

    if send:
        # step 4: sending sequences
        old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
        environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass

        LogEntry.create('Runtime',
                        ("Submitting sequences for pre_processed_id: "
                         "%d" % preprocessed_data_id))
        try:
            for cmd in ebi_submission.generate_send_sequences_cmd():
                try:
                    stdout, stderr, _ = system_call(cmd)
                except Exception as e:
                    stdout = ''
                    stderr = str(e)
                    le = LogEntry.create(
                        'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)),
                        info={'ebi_submission': preprocessed_data_id})
                    ebi_submission.study.ebi_submission_status = (
                        "failed: ASCP submission, log id: %d" % le.id)
                    raise ComputeError("EBI Submission failed! Log id: "
                                       "%d" % le.id)
                finally:
                    open(ebi_submission.ascp_reply, 'a').write(
                        'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
        finally:
            environ['ASPERA_SCP_PASS'] = old_ascp_pass
        LogEntry.create('Runtime',
                        ('Submission of sequences of pre_processed_id: '
                         '%d completed successfully' %
                         preprocessed_data_id))

        # step 5: sending xml and parsing answer
        xmls_cmds = ebi_submission.generate_curl_command()
        LogEntry.create('Runtime',
                        ("Submitting XMLs for pre_processed_id: "
                         "%d" % preprocessed_data_id))
        try:
            xml_content, stderr, _ = system_call(xmls_cmds)
        except Exception as e:
            xml_content = ''
            stderr = str(e)
            le = LogEntry.create('Fatal',
                                 "Command: %s\nError: %s\n" % (
                                    cmd, str(e)),
                                 info={'ebi_submission': preprocessed_data_id})
            ebi_submission.study.ebi_submission_status = (
                "failed: XML submission, log id: %d" % le.id)
            raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
        else:
            LogEntry.create('Runtime',
                            ('Submission of sequences of pre_processed_id: '
                             '%d completed successfully' %
                             preprocessed_data_id))
        finally:
            open(ebi_submission.curl_reply, 'w').write(
                'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))

        try:
            st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
                ebi_submission.parse_EBI_reply(xml_content)
        except EBISubmissionError as e:
            le = LogEntry.create(
                'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
                info={'ebi_submission': preprocessed_data_id})
            ebi_submission.study.ebi_submission_status = (
                "failed: XML parsing, log id: %d" % le.id)
            raise ComputeError("EBI Submission failed! Log id: %d" % le.id)

#.........这里部分代码省略.........

# 需要导入模块: from qiita_ware.ebi import EBISubmission [as 别名]
# 或者: from qiita_ware.ebi.EBISubmission import generate_demultiplexed_fastq [as 别名]
def submit_EBI(artifact_id, action, send, test=False):
    """Submit an artifact to EBI

    Parameters
    ----------
    artifact_id : int
        The artifact id
    action : %s
        The action to perform with this data
    send : bool
        True to actually send the files
    test : bool
        If True some restrictions will be ignored, only used in parse_EBI_reply
    """
    # step 1: init and validate
    ebi_submission = EBISubmission(artifact_id, action)

    # step 2: generate demux fastq files
    try:
        ebi_submission.generate_demultiplexed_fastq()
    except Exception:
        error_msg = format_exc()
        if isdir(ebi_submission.full_ebi_dir):
            rmtree(ebi_submission.full_ebi_dir)
        LogEntry.create('Runtime', error_msg,
                        info={'ebi_submission': artifact_id})
        raise

    # step 3: generate and write xml files
    ebi_submission.generate_xml_files()

    if send:
        # getting aspera's password
        old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
        if old_ascp_pass == '':
            environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
        ascp_passwd = environ['ASPERA_SCP_PASS']
        LogEntry.create('Runtime',
                        ('Submission of sequences of pre_processed_id: '
                         '%d completed successfully' % artifact_id))

        # step 4: sending sequences
        if action != 'MODIFY':
            LogEntry.create('Runtime',
                            ("Submitting sequences for pre_processed_id: "
                             "%d" % artifact_id))
            for cmd in ebi_submission.generate_send_sequences_cmd():
                stdout, stderr, rv = system_call(cmd)
                if rv != 0:
                    error_msg = ("ASCP Error:\nStd output:%s\nStd error:%s" % (
                        stdout, stderr))
                    environ['ASPERA_SCP_PASS'] = old_ascp_pass
                    raise ComputeError(error_msg)
                open(ebi_submission.ascp_reply, 'a').write(
                    'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
        environ['ASPERA_SCP_PASS'] = old_ascp_pass

        # step 5: sending xml and parsing answer
        xmls_cmds = ebi_submission.generate_curl_command(
            ebi_seq_xfer_pass=ascp_passwd)
        LogEntry.create('Runtime',
                        ("Submitting XMLs for pre_processed_id: "
                         "%d" % artifact_id))
        xml_content, stderr, rv = system_call(xmls_cmds)
        if rv != 0:
            error_msg = ("Error:\nStd output:%s\nStd error:%s" % (
                xml_content, stderr))
            raise ComputeError(error_msg)
        else:
            LogEntry.create('Runtime',
                            ('Submission of sequences of pre_processed_id: '
                             '%d completed successfully' % artifact_id))
        open(ebi_submission.curl_reply, 'w').write(
            'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))

        try:
            st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
                ebi_submission.parse_EBI_reply(xml_content, test=test)
        except EBISubmissionError as e:
            error = str(e)
            le = LogEntry.create(
                'Fatal', "Command: %s\nError: %s\n" % (xml_content, error),
                info={'ebi_submission': artifact_id})
            raise ComputeError(
                "EBI Submission failed! Log id: %d\n%s" % (le.id, error))

        if action == 'ADD' or test:
            if st_acc:
                ebi_submission.study.ebi_study_accession = st_acc
            if sa_acc:
                ebi_submission.sample_template.ebi_sample_accessions = sa_acc
            if bio_acc:
                ebi_submission.sample_template.biosample_accessions = bio_acc
            if ex_acc:
                ebi_submission.prep_template.ebi_experiment_accessions = ex_acc
            ebi_submission.artifact.ebi_run_accessions = run_acc
    else:
        st_acc, sa_acc, bio_acc, ex_acc, run_acc = None, None, None, None, None

    return st_acc, sa_acc, bio_acc, ex_acc, run_acc