Python CharonSession.sample_update方法代码示例

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def write_to_charon_NGI_results(job_id, return_code, run_dir):
    """Update the status of a sequencing run after alignment.

    :param NGIProject project_id: The name of the project, sample, lib prep, flowcell id
    :param int return_code: The return code of the workflow process
    :param string run_dir: the directory where results are stored (I know that I am running piper)

    :raises RuntimeError: If the Charon database could not be updated
    """
    charon_session = CharonSession()
    # Consider moving this mapping to the CharonSession object or something
    if return_code is None:
        status = "RUNNING"
    elif return_code == 0:
        status = "DONE"
    else:
        ## TODO we need to differentiate between COMPUTATION_FAILED and DATA_FAILED
        ##      also there is IGNORE?
        status = "COMPUTATION_FAILED"
    try:
        m_dict = STHLM_UUSNP_SAMPLE_RE.match(job_id).groupdict()
        #m_dict = re.match(r'?P<project_name>\w\.\w+_\d+_\d+|\w{2}-\d+)_(?P<sample_id>[\w-]+)_(?P<libprep_id>\w|\w{2}\d{3}_\2)_(?P<seqrun_id>\d{6}_\w+_\d{4}_.{10})', job_id).groupdict()
        project_id = get_project_id_from_name(m_dict['project_name'])
        sample_id = m_dict['sample_id']
    except (TypeError, AttributeError):
        error_msg = "Could not parse project/sample ids from job id \"{}\"; cannot update Charon with results!".format(job_id)
        raise RuntimeError(error_msg)
    try:
        charon_session.sample_update(project_id, sample_id, status=status)
    except CharonError as e:
        error_msg = ('Failed to update sample status to "{}" for sample "{}" '
                     'in Charon database: {}'.format(status, project_id, sample_id, e))
        raise RuntimeError(error_msg)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def update_gtc_for_sample(project_id, sample_id, piper_gtc_path, config=None, config_file_path=None):
    """Find the genotype concordance file for this sample, if it exists,
    and update the sample record in Charon with the value parsed from it.

    :param str project_id: The id of the project
    :param str sample_id: The id the sample
    :param str piper_gtc_path: The path to the piper genotype concordance directory

    :raises CharonError: If there is some Error -- with Charon
    :raises IOError: If the path specified is missing or inaccessible
    :raises ValueError: If the specified sample has no data in the gtc file
    """
    gtc_file = os.path.join(piper_gtc_path, "{}.gt_concordance".format(sample_id))
    try:
        concordance_value = parse_genotype_concordance(gtc_file)[sample_id]
    except KeyError:
        raise ValueError('Concordance data for sample "{}" not found in gt '
                         'concordance file "{}"'.format(sample_id, gtc_file))
    gtc_lower_bound = config.get("genotyping", {}).get("lower_bound_cutoff")
    status_dict = {}
    if gtc_lower_bound:
        if concordance_value < concordance_value:
            status_dict = {"genotype_status": "FAILED"}
        else:
            status_dict = {"genotype_status": "PASSED"}
    charon_session = CharonSession()
    charon_session.sample_update(projectid=project_id, sampleid=sample_id,
                                 genotype_concordance=concordance_value,
                                 **status_dict)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def update_gt_status_in_charon(sample_id, status, concordance=None):
    project_id = sample_id.split('_')[0]
    try:
        charon_session = CharonSession()
        sample = charon_session.sample_get(project_id, sample_id)
        if concordance is None:
            if sample.get('genotype_status') != status:
                charon_session.sample_update(projectid=project_id, sampleid=sample_id,genotype_status=status)
        else:
            if sample.get('genotype_status') != status or sample.get('genotype_concordance') != concordance:
                charon_session.sample_update(projectid=project_id, sampleid=sample_id,genotype_status=status, genotype_concordance=concordance)
    except CharonError as e:
        return str(e)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def main(inbox=None, num_days=14, genotype_files=None, config=None, config_file_path=None):
    if genotype_files:
        gt_files_valid = [os.path.abspath(gt_file) for gt_file in genotype_files]
    else:
        if not inbox:
            try:
                inboxes = config["environment"]["flowcell_inbox"]
            except (KeyError, TypeError):
                raise ValueError("No path to delivery inbox specified by argument "
                                 "or in configuration file ({}). Exiting.".format(config_file_path))
        for inbox in inboxes:
            inbox = os.path.abspath(inbox)
            # Convert to seconds
            cutoff_age = time.time() - (int(num_days) * 24 * 60 * 60)
            LOG.info("Searching for genotype files under {} modified after "
                     "{}".format(inbox, time.ctime(cutoff_age)))
            gt_files_valid = []
            for gt_file in filter(GENOTYPE_FILE_RE.match, glob.glob(os.path.join(inbox, "*"))):
                if os.stat(gt_file).st_mtime > time.time() - cutoff_age:
                    gt_files_valid.append(os.path.abspath(gt_file))

    if not gt_files_valid:
        LOG.info("No genotype files found under {} newer than "
                 "{}".format(inbox, time.ctime(cutoff_age)))
    else:
        charon_session = CharonSession()
        for gt_file_path in gt_files_valid:
            project_samples_dict = \
                    find_projects_from_samples(parse_samples_from_vcf(gt_file_path))
            for project_id, samples in project_samples_dict.iteritems():
                LOG.info("Updating project {}...".format(project_id))
                for sample in samples:
                    try:
                        genotype_status = \
                            charon_session.sample_get(projectid=project_id,
                                                      sampleid=sample).get("genotype_status")
                        if genotype_status in (None, "NOT_AVAILABLE"):
                            LOG.info('Updating sample {} genotype_status '
                                     'to "AVAILABLE"...'.format(sample))
                            charon_session.sample_update(projectid=project_id,
                                                         sampleid=sample,
                                                         genotype_status="AVAILABLE")
                        else:
                            LOG.info('Not updating sample {} genotype_status '
                                     '(already "{}")'.format(sample, genotype_status))
                    except CharonError as e:
                        LOG.error('Could not update genotype status to "AVAILABLE" '
                                  'for project/sample "{}/{}": {}'.format(project_id,
                                                                          sample,
                                                                          e))

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def main(project):

    charon_session = CharonSession()
    samples = charon_session.project_get_samples(project)
    for sample in samples["samples"]:
        
        charon_session.sample_update(project, sample["sampleid"],
                                     analysis_status= "TO_ANALYZE",
                                     genotype_status=None,
                                     total_autosomal_coverage="0",
                                     total_sequenced_reads="0")
        for sample_prep in charon_session.sample_get_libpreps(project, sample["sampleid"])['libpreps']:
            seqruns = charon_session.libprep_get_seqruns(project, sample["sampleid"], sample_prep["libprepid"])['seqruns']
            for seqrun in seqruns:
                charon_session.seqrun_update(project, sample["sampleid"], sample_prep["libprepid"], seqrun["seqrunid"],
                                             mean_autosomal_coverage = "0",
                                             alignment_status  = "NOT_RUNNING")

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
 def add_supr_name_delivery_in_charon(self, supr_name_of_delivery):
     '''Updates delivery_projects in Charon at project level
     '''
     charon_session = CharonSession()
     try:
         #fetch the project
         sample_charon = charon_session.sample_get(self.projectid, self.sampleid)
         delivery_projects = sample_charon['delivery_projects']
         if supr_name_of_delivery not in sample_charon:
             delivery_projects.append(supr_name_of_delivery)
             charon_session.sample_update(self.projectid, self.sampleid, delivery_projects=delivery_projects)
             logger.info('Charon delivery_projects for sample {} updated with value {}'.format(self.sampleid, supr_name_of_delivery))
         else:
             logger.warn('Charon delivery_projects for sample {} not updated with value {} because the value was already present'.format(self.sampleid, supr_name_of_delivery))
     except Exception, e:
         logger.error('Failed to update delivery_projects in charon while delivering {}. Error says: {}'.format(self.sampleid, e))
         logger.exception(e)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def update_sample_duplication_and_coverage(project_id, sample_id, project_base_path,
                                       config=None, config_file_path=None):
    """Update Charon with the duplication rates for said sample.

    :param str project_base_path: The path to the project dir 
    :param str sample_id: The sample name (e.g. P1170_105)

    """
    
    dup_file_path=os.path.join(project_base_path, 'ANALYSIS', project_id, 'piper_ngi', '05_processed_alignments', "{}.metrics".format(sample_id))
    genome_results_file_path=os.path.join(project_base_path, 'ANALYSIS', project_id, 'piper_ngi', '06_final_alignment_qc', "{}.clean.dedup.qc".format(sample_id),"genome_results.txt")

    try:
        dup_pc=parse_deduplication_percentage(dup_file_path)
    except:
        dup_pc=0
        LOG.error("Cannot find {}.metrics file for duplication rate at {}. Continuing.".format(sample_id, dup_file_path))
    try:
        cov=parse_qualimap_coverage(genome_results_file_path)
        reads=parse_qualimap_reads(genome_results_file_path)
    except IOError as e:
        cov=0
        reads=0
        LOG.error("Cannot find genome_results.txt file for sample coverage at {}. Continuing.".format(genome_results_file_path))
    try:
        charon_session = CharonSession()
        charon_session.sample_update(projectid=project_id,
                                     sampleid=sample_id,
                                     duplication_pc=dup_pc,
                                     total_sequenced_reads=reads,
                                     total_autosomal_coverage=cov)
        LOG.info('Updating sample "{}" in '
                 'Charon with mean duplication_percentage"{}" and autosomal coverage "{}"'.format(sample_id, dup_pc, cov))
    except CharonError as e:
        error_text = ('Could not update project/sample "{}/{}" '
                    'in Charon with duplication rate : {}'
                      'and coverage {}'.format("{}/{}".format(project_id, sampleid, dup_pc, cov)))
        LOG.error(error_text)
        if not config.get('quiet'):
            mail_analysis(project_name=project_id, sample_name=sample_id,
                          engine_name="piper_ngi", level="ERROR", info_text=error_text)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def kill_running_sample_analysis(workflow_subtask, project_id, sample_id):
    """Determine if a sample is currently being analyzed by accessing the local
    process tracking database."""
    sample_run_name = "{}/{}".format(project_id, sample_id)
    LOG.info('Attempting to kill sample analysis run "{}"'.format(sample_run_name))
    LOG.info('Checking if sample run "{}" is currently being analyzed '
             '(workflow "{}")...'.format(sample_run_name, workflow_subtask))
    with get_db_session() as session:
        db_q = session.query(SampleAnalysis).filter_by(workflow=workflow_subtask,
                                                       project_id=project_id,
                                                       sample_id=sample_id)
        sample_run = db_q.first()
        if sample_run:
            try:
                slurm_job_id = sample_run.slurm_job_id
                LOG.info('...sample run "{}" is currently being analyzed '
                         '(workflow subtask "{}") and has slurm job id "{}"; '
                         'trying to kill it...'.format(sample_run_name,
                                                       workflow_subtask,
                                                       slurm_job_id))
                kill_slurm_job_by_id(slurm_job_id)
            except Exception as e:
                LOG.error('Could not kill sample run "{}": {}'.format(sample_run_name, e))
                return False
            try:
                project_obj = create_project_obj_from_analysis_log(sample_run.project_name,
                                                                   sample_run.project_id,
                                                                   sample_run.project_base_path,
                                                                   sample_run.sample_id,
                                                                   sample_run.workflow)
            except IOError as e: # analysis log file is missing!
                error_text = ('Could not find analysis log file! Cannot update '
                              'Charon for {} run {}/{}: {}'.format(sample_run.workflow,
                                                                   sample_run.project_id,
                                                                   sample_run.sample_id,
                                                                   e))
                LOG.error(error_text)
            else:
                try:
                    charon_session = CharonSession()
                    set_status = "FAILED"
                    if workflow_subtask == "genotype_concordance":
                        status_field = "genotype_status"
                    elif workflow_subtask == "merge_process_variantcall":
                        sample_status_field = "analysis_status"
                        seqrun_status_field = "alignment_status"
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 **{sample_status_field: set_status})
                    recurse_status_for_sample(project_obj,
                                              status_field=seqrun_status_field,
                                              status_value=set_status)
                except CharonError as e:
                    LOG.error('Couldn\'t update Charon field "{}" to "{} for '
                              'project/sample "{}/{}"'.format(status_field, set_status,
                                                              project_id, sample_id))
            try:
                LOG.info('Removing sample run "{}" from local jobs database...'.format(sample_run_name))
                # Remove from local jobs database
                session.delete(sample_run)
                session.commit()
                LOG.info("Deleted.")
            except Exception as e:
                LOG.error('Failed to remove entry for sample run "{}" from '
                          'local jobs database: {}'.format(sample_run_name, e))
        else:
            LOG.info('...sample run "{}" is not currently under analysis.'.format(sample_run_name))
    return True

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def update_charon_with_local_jobs_status(quiet=False, config=None, config_file_path=None):
    """Check the status of all locally-tracked jobs and update Charon accordingly.
    """
    if quiet and not config.get("quiet"):
        config['quiet'] = True
    LOG.info("Updating Charon with the status of all locally-tracked jobs...")
    with get_db_session() as session:
        charon_session = CharonSession()
        for sample_entry in session.query(SampleAnalysis).all():
            # Local names
            workflow = sample_entry.workflow
            project_name = sample_entry.project_name
            project_id = sample_entry.project_id
            project_base_path = sample_entry.project_base_path
            sample_id = sample_entry.sample_id
            engine = sample_entry.engine
            # Only one of these id fields (slurm, pid) will have a value
            slurm_job_id = sample_entry.slurm_job_id
            process_id = sample_entry.process_id
            piper_exit_code = get_exit_code(workflow_name=workflow,
                                            project_base_path=project_base_path,
                                            project_name=project_name,
                                            project_id=project_id,
                                            sample_id=sample_id)
            label = "project/sample {}/{}".format(project_name, sample_id)

            if workflow not in ("merge_process_variantcall", "genotype_concordance",):
                LOG.error('Unknown workflow "{}" for {}; cannot update '
                          'Charon. Skipping sample.'.format(workflow, label))
                continue

            try:
                project_obj = create_project_obj_from_analysis_log(project_name,
                                                                   project_id,
                                                                   project_base_path,
                                                                   sample_id,
                                                                   workflow)
            except IOError as e: # analysis log file is missing!
                error_text = ('Could not find analysis log file! Cannot update '
                              'Charon for {} run {}/{}: {}'.format(workflow,
                                                                   project_id,
                                                                   sample_id,
                                                                   e))
                LOG.error(error_text)
                if not config.get('quiet'):
                    mail_analysis(project_name=project_name,
                                  sample_name=sample_id,
                                  engine_name=engine,
                                  level="ERROR",
                                  info_text=error_text,
                                  workflow=workflow)
                continue
            try:
                if piper_exit_code == 0:
                    # 0 -> Job finished successfully
                    if workflow == "merge_process_variantcall":
                        sample_status_field = "analysis_status"
                        seqrun_status_field = "alignment_status"
                        set_status = "ANALYZED" # sample level
                    elif workflow == "genotype_concordance":
                        sample_status_field = seqrun_status_field = "genotype_status"
                        set_status = "DONE" # sample level
                    recurse_status = "DONE" # For the seqrun level
                    info_text = ('Workflow "{}" for {} finished succesfully. '
                                 'Recording status {} in Charon'.format(workflow,
                                                                        label,
                                                                        set_status))
                    LOG.info(info_text)
                    if not config.get('quiet'):
                        mail_analysis(project_name=project_name,
                                      sample_name=sample_id,
                                      engine_name=engine,
                                      level="INFO",
                                      info_text=info_text,
                                      workflow=workflow)
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 **{sample_status_field: set_status})
                    recurse_status_for_sample(project_obj,
                                              status_field=seqrun_status_field,
                                              status_value=recurse_status,
                                              config=config)
                    # Job is only deleted if the Charon status update succeeds
                    session.delete(sample_entry)
                    #run MultiQC
                    LOG.info("Running MultiQC on project {}".format(project_name))
                    try:
                        run_multiqc(project_base_path, project_id, project_name)
                    except Exception as e:
                        LOG.error(e)


                    if workflow == "merge_process_variantcall":
                        # Parse seqrun output results / update Charon
                        # This is a semi-optional step -- failure here will send an
                        # email but not more than once. The record is still removed
                        # from the local jobs database, so this will have to be done
                        # manually if you want it done at all.
                        piper_qc_dir = os.path.join(project_base_path, "ANALYSIS",
                                                    project_id, "piper_ngi",
#.........这里部分代码省略.........

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def update_charon_with_local_jobs_status(config=None, config_file_path=None):
    """Check the status of all locally-tracked jobs and update Charon accordingly.
    """
    LOG.info("Updating Charon with the status of all locally-tracked jobs...")
    with get_db_session() as session:
        charon_session = CharonSession()
        for sample_entry in session.query(SampleAnalysis).all():
            # Local names
            workflow = sample_entry.workflow
            project_name = sample_entry.project_name
            project_id = sample_entry.project_id
            project_base_path = sample_entry.project_base_path
            sample_id = sample_entry.sample_id
            engine=sample_entry.engine
            # Only one of these will have a value
            slurm_job_id = sample_entry.slurm_job_id
            process_id = sample_entry.process_id
            piper_exit_code = get_exit_code(workflow_name=workflow,
                                            project_base_path=project_base_path,
                                            project_name=project_name,
                                            project_id=project_id,
                                            sample_id=sample_id)
            label = "project/sample {}/{}".format(project_name, sample_id)

            try:
                project_obj = create_project_obj_from_analysis_log(project_name,
                                                                   project_id,
                                                                   project_base_path,
                                                                   sample_id,
                                                                   workflow)
            except IOError as e: # analysis log file is missing!
                error_text = ('Could not find analysis log file! Cannot update '
                              'Charon for sample run {}/{}: {}'.format(project_id,
                                                                   sample_id,
                                                                   e))
                LOG.error(error_text)
                if not config.get('quiet'):
                    mail_analysis(project_name=project_name, sample_name=sample_id,
                              engine_name=engine, level="ERROR", info_text=error_text)
                continue
            try:
                if piper_exit_code and piper_exit_code == 0:
                    # 0 -> Job finished successfully
                    set_status = "ANALYZED"
                    info_text = ('Workflow "{}" for {} finished succesfully. '
                                 'Recording status {} in Charon'.format(workflow, label,
                                                                        set_status))
                    LOG.info(info_text)
                    if not config.get('quiet'):
                        mail_analysis(project_name=project_name, sample_name=sample_id,
                                  engine_name=engine, level="INFO", info_text=info_text)
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 analysis_status=set_status)
                    recurse_status="DONE"
                    recurse_status_for_sample(project_obj, recurse_status)
                    # Job is only deleted if the Charon status update succeeds
                    session.delete(sample_entry)
                    # Parse seqrun output results / update Charon
                    # This is a semi-optional step -- failure here will send an
                    # email but not more than once. The record is still removed
                    # from the local jobs database, so this will have to be done
                    # manually if you want it done at all.
                    piper_qc_dir = os.path.join(project_base_path, "ANALYSIS",
                                                project_id,"piper_ngi",  "02_preliminary_alignment_qc")
                    update_coverage_for_sample_seqruns(project_id, sample_id, piper_qc_dir)
                elif piper_exit_code and piper_exit_code >0:
                    # 1 -> Job failed
                    set_status = "FAILED"
                    error_text = ('Workflow "{}" for {} failed. Recording status '
                                 '{} in Charon.'.format(workflow, label, set_status))
                    LOG.error(error_text)
                    if not config.get('quiet'):
                        mail_analysis(project_name=project_name, sample_name=sample_id,
                                  engine_name=engine, level="ERROR", info_text=error_text)
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 analysis_status=set_status)
                    recurse_status_for_sample(project_obj, set_status)
                    # Job is only deleted if the Charon update succeeds
                    session.delete(sample_entry)
                else:
                    # None -> Job still running OR exit code was never written (failure)
                    JOB_FAILED = None
                    if slurm_job_id:
                        try:
                            slurm_exit_code = get_slurm_job_status(slurm_job_id)
                        except ValueError as e:
                            slurm_exit_code = 1
                        if slurm_exit_code is not None: # "None" indicates job is still running
                            JOB_FAILED = True
                    else:
                        if not psutil.pid_exists(process_id):
                            # Job did not write an exit code and is also not running
                            JOB_FAILED = True
                    if JOB_FAILED:
                        set_status = "FAILED"
                        error_text = ('No exit code found but job not running for '
                                      '{}: setting status to {} in Charon'.format(label, set_status))
                        LOG.error(error_text)
#.........这里部分代码省略.........

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def create_charon_entries_from_project(project, best_practice_analysis="whole_genome_reseq",
                                       sequencing_facility="NGI-S",
                                       force_overwrite=False, delete_existing=False,
                                       retry_on_fail=True):
    """Given a project object, creates the relevant entries in Charon.
    This code is remarkably shoddy as I created it in a hurry and then later
    it became a part of the pipeline. Use at your own risk! Ha ha.

    :param NGIProject project: The NGIProject object
    :param str best_practice_analysis: The workflow to assign for this project (default "variant_calling")
    :param str sequencing_facility: The facility that did the sequencing
    :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false)
    :param bool delete_existing: Don't just update existing entries, delete them and create new ones (default false)
    """
    charon_session = CharonSession()
    update_failed=False
    try:
        status = "OPEN"
        LOG.info('Creating project "{}" with status "{}", best practice analysis "{}", '
                 'and sequencing_facility {}'.format(project, status,
                                                     best_practice_analysis,
                                                     sequencing_facility))
        charon_session.project_create(projectid=project.project_id,
                                      name=project.name,
                                      status=status,
                                      best_practice_analysis=best_practice_analysis,
                                      sequencing_facility=sequencing_facility)
        LOG.info('Project "{}" created in Charon.'.format(project))
    except CharonError as e:
        if e.status_code == 400:
            if force_overwrite:
                LOG.warn('Overwriting data for project "{}"'.format(project))
                charon_session.project_update(projectid=project.project_id,
                                              name=project.name,
                                              status=status,
                                              best_practice_analysis=best_practice_analysis,
                                              sequencing_facility=sequencing_facility)
                LOG.info('Project "{}" updated in Charon.'.format(project))
            else:
                LOG.info('Project "{}" already exists; moving to samples...'.format(project))
        else:
            raise
    for sample in project:
        if delete_existing:
            LOG.warn('Deleting existing sample "{}"'.format(sample))
            try:
                charon_session.sample_delete(projectid=project.project_id,
                                             sampleid=sample.name)
            except CharonError as e:
                update_failed=True
                LOG.error('Could not delete sample "{}": {}'.format(sample, e))
        try:
            analysis_status = "TO_ANALYZE"
            LOG.info('Creating sample "{}" with analysis_status "{}"'.format(sample, analysis_status))
            charon_session.sample_create(projectid=project.project_id,
                                         sampleid=sample.name,
                                         analysis_status=analysis_status)
            LOG.info('Project/sample "{}/{}" created in Charon.'.format(project, sample))
        except CharonError as e:
            if e.status_code == 400:
                if force_overwrite:
                    LOG.warn('Overwriting data for project "{}" / '
                             'sample "{}"'.format(project, sample))
                    charon_session.sample_update(projectid=project.project_id,
                                                 sampleid=sample.name,
                                                 analysis_status=analysis_status)
                    LOG.info('Project/sample "{}/{}" updated in Charon.'.format(project, sample))
                else:
                    LOG.info('Project "{}" / sample "{}" already exists; moving '
                             'to libpreps'.format(project, sample))
            else:
                update_failed=True
                LOG.error(e)
                continue
        for libprep in sample:
            if delete_existing:
                LOG.warn('Deleting existing libprep "{}"'.format(libprep))
                try:
                    charon_session.libprep_delete(projectid=project.project_id,
                                                  sampleid=sample.name,
                                                  libprepid=libprep.name)
                except CharonError as e:
                    LOG.warn('Could not delete libprep "{}": {}'.format(libprep, e))
            try:
                qc = "PASSED"
                LOG.info('Creating libprep "{}" with qc status "{}"'.format(libprep, qc))
                charon_session.libprep_create(projectid=project.project_id,
                                              sampleid=sample.name,
                                              libprepid=libprep.name,
                                              qc=qc)
                LOG.info(('Project/sample/libprep "{}/{}/{}" created in '
                          'Charon').format(project, sample, libprep))
            except CharonError as e:
                if e.status_code == 400:
                    if force_overwrite:
                        LOG.warn('Overwriting data for project "{}" / '
                                 'sample "{}" / libprep "{}"'.format(project, sample,
                                                                     libprep))
                        charon_session.libprep_update(projectid=project.project_id,
                                                      sampleid=sample.name,
#.........这里部分代码省略.........

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]

#.........这里部分代码省略.........
                                                 sampleid=sample_id,
                                                 libprepid=libprep_id,
                                                 seqrunid=seqrun_id,
                                                 alignment_status="FAILED")
                    # Job is only deleted if the Charon update succeeds
                    LOG.debug("Deleting local entry {}".format(seqrun_entry))
                    session.delete(seqrun_entry)
                else:
                    # None -> Job still running
                    charon_status = charon_session.seqrun_get(projectid=project_id,
                                                              sampleid=sample_id,
                                                              libprepid=libprep_id,
                                                              seqrunid=seqrun_id)['alignment_status']
                    if not charon_status == "RUNNING":
                        LOG.warn('Tracking inconsistency for {}: Charon status is "{}" but '
                                 'local process tracking database indicates it is running. '
                                 'Setting value in Charon to RUNNING.'.format(label, charon_status))
                        charon_session.seqrun_update(projectid=project_id,
                                                     sampleid=sample_id,
                                                     libprepid=libprep_id,
                                                     seqrunid=seqrun_id,
                                                     alignment_status="RUNNING")
            except CharonError as e:
                LOG.error('Unable to update Charon status for "{}": {}'.format(label, e))


        for sample_entry in session.query(SampleAnalysis).all():

            # Local names
            workflow = sample_entry.workflow
            project_name = sample_entry.project_name
            project_id = sample_entry.project_id
            project_base_path = sample_entry.project_base_path
            sample_id = sample_entry.sample_id
            pid = sample_entry.process_id

            exit_code = get_exit_code(workflow_name=workflow,
                                      project_base_path=project_base_path,
                                      project_name=project_name,
                                      sample_id=sample_id)
            label = "project/sample/libprep/seqrun {}/{}".format(project_name,
                                                                       sample_id)
            try:
                if exit_code == 0:
                    # 0 -> Job finished successfully
                    LOG.info('Workflow "{}" for {} finished succesfully. '
                             'Recording status "DONE" in Charon'.format(workflow, label))
                    set_status = "DONE"
                    ## TODO implement sample-level analysis results parsing / reporting to Charon?
                    #try:
                    #    write_to_charon_alignment_results(base_path=project_base_path,
                    #                                      project_name=project_name,
                    #                                      project_id=project_id,
                    #                                      sample_id=sample_id,
                    #                                      libprep_id=libprep_id,
                    #                                      seqrun_id=seqrun_id)
                    #except (RuntimeError, ValueError) as e:
                    #    LOG.error(e)
                    #    set_alignment_status = "FAILED"
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 status=set_status)
                    # Job is only deleted if the Charon update succeeds
                    session.delete(sample_entry)
                elif exit_code == 1 or (not psutil.pid_exists(pid) and not exit_code):
                    if exit_code == 1:
                        # 1 -> Job failed (DATA_FAILURE / COMPUTATION_FAILURE ?)
                        LOG.info('Workflow "{}" for {} failed. Recording status '
                                 '"COMPUTATION_FAILED" in Charon.'.format(workflow, label))
                    else:
                        # Job failed without writing an exit code
                        LOG.error('ERROR: No exit code found for process {} '
                                  'but it does not appear to be running '
                                  '(pid {} does not exist). Setting status to '
                                  '"COMPUTATION_FAILED", inspect manually'.format(label, pid))
                    charon_session.sample_update(projectid=project_id,
                                                 sampleid=sample_id,
                                                 status="COMPUTATION_FAILED")
                    # Job is only deleted if the Charon update succeeds
                    session.delete(sample_entry)
                else:
                    # None -> Job still running
                    try:
                        charon_status = charon_session.sample_get(projectid=project_id,
                                                              sampleid=sample_id)['status']
                    except (CharonError, KeyError) as e:
                        LOG.warn('Unable to get required information from Charon for '
                          'sample "{}" / project "{}" -- forcing it to RUNNING: {}'.format(sample_id, project_id, e))
                        charon_status = "NEW"

                    if not charon_status == "RUNNING":
                        LOG.warn('Tracking inconsistency for {}: Charon status is "{}" but '
                                 'local process tracking database indicates it is running. '
                                 'Setting value in Charon to RUNNING.'.format(label, charon_status))
                        charon_session.sample_update(projectid=project_id,
                                                     sampleid=sample_id,
                                                     status="RUNNING")
            except CharonError as e:
                LOG.error('Unable to update Charon status for "{}": {}'.format(label, e))
        session.commit()

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def create_charon_entries_from_project(project, workflow="NGI", force_overwrite=False):
    """Given a project object, creates the relevant entries
    in Charon.

    :param NGIProject project: The NGIProject object
    :param str workflow: The workflow to assign for this project (default NGI)
    :param bool force_overwrite: If this is set to true, overwrite existing entries in Charon (default false)
    """
    charon_session = CharonSession()
    try:
        status="SEQUENCED"
        LOG.info('Creating project "{}" with status "{}" and workflow "{}"'.format(project, status, workflow))
        charon_session.project_create(projectid=project.project_id,
                                      name=project.name,
                                      status=status,
                                      pipeline=workflow)
    except CharonError:
        if force_overwrite:
            LOG.warn('Overwriting data for project "{}"'.format(project))
            charon_session.project_update(projectid=project.project_id,
                                          name=project.name,
                                          status=status,
                                          pipeline=workflow)
        else:
            LOG.info('Project "{}" already exists; moving to samples...'.format(project))

    for sample in project:
        try:
            LOG.info('Creating sample "{}"'.format(sample))
            charon_session.sample_create(projectid=project.project_id,
                                         sampleid=sample.name,
                                         status="NEW")
        except CharonError:
            if force_overwrite:
                LOG.warn('Overwriting data for project "{}" / '
                         'sample "{}"'.format(project, sample))
                charon_session.sample_update(projectid=project.project_id,
                                             sampleid=sample.name,
                                             status="NEW")
            else:
                LOG.info('Project "{}" / sample "{}" already exists; moving '
                         'to libpreps'.format(project, sample))

        for libprep in sample:
            try:
                LOG.info('Creating libprep "{}"'.format(libprep))
                charon_session.libprep_create(projectid=project.project_id,
                                              sampleid=sample.name,
                                              libprepid=libprep.name,
                                              status="NEW")
            except CharonError:
                if force_overwrite:
                    LOG.warn('Overwriting data for project "{}" / '
                             'sample "{}" / libprep "{}"'.format(project, sample,
                                                                 libprep))
                    charon_session.libprep_update(projectid=project.project_id,
                                                  sampleid=sample.name,
                                                  libprepid=libprep.name,
                                                  status="NEW")
                else:
                    LOG.info('Project "{}" / sample "{}" / libprep "{}" already '
                             'exists; moving to libpreps'.format(project, sample, libprep))

            for seqrun in libprep:
                try:
                    LOG.info('Creating seqrun "{}"'.format(seqrun))
                    charon_session.seqrun_create(projectid=project.project_id,
                                                 sampleid=sample.name,
                                                 libprepid=libprep.name,
                                                 seqrunid=seqrun.name,
                                                 total_reads=0,
                                                 mean_autosomal_coverage=0,
                                                 sequencing_status="DONE",
                                                 alignment_status="NEW")
                except CharonError as e:
                    if force_overwrite:
                        LOG.warn('Overwriting data for project "{}" / '
                                 'sample "{}" / libprep "{}" / '
                                 'seqrun "{}"'.format(project, sample,
                                                      libprep, seqrun))
                        charon_session.seqrun_update(projectid=project.project_id,
                                                     sampleid=sample.name,
                                                     libprepid=libprep.name,
                                                     seqrunid=seqrun.name,
                                                     status="NEW")
                    else:
                        LOG.info('Project "{}" / sample "{}" / libprep "{}" / '
                                 'seqrun "{}" already exists; next...'.format(project, sample,
                                                                              libprep, seqrun))

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
 def save_delivery_token_in_charon(self, delivery_token):
     '''Updates delivery_token in Charon at sample level
     '''
     charon_session = CharonSession()
     charon_session.sample_update(self.projectid, self.sampleid, delivery_token=delivery_token)

# 需要导入模块: from ngi_pipeline.database.classes import CharonSession [as 别名]
# 或者: from ngi_pipeline.database.classes.CharonSession import sample_update [as 别名]
def launch_analysis(level, projects_to_analyze, restart_failed_jobs=False,
                    config=None, config_file_path=None):
    """Launch the appropriate seqrun (flowcell-level) analysis for each fastq
    file in the project.

    :param list projects_to_analyze: The list of projects (Project objects) to analyze
    :param dict config: The parsed NGI configuration file; optional/has default.
    :param str config_file_path: The path to the NGI configuration file; optional/has default.
    """
    # Update Charon with the local state of all the jobs we're running
    update_charon_with_local_jobs_status()
    charon_session = CharonSession()
    for project in projects_to_analyze:
        # Get information from Charon regarding which workflows to run
        try:
            # E.g. "NGI" for NGI DNA Samples
            workflow = charon_session.project_get(project.project_id)["pipeline"]
        except (KeyError, CharonError) as e:
            # Workflow missing from Charon?
            LOG.error('Skipping project "{}" because of error: {}'.format(project, e))
            continue
        try:
            analysis_engine_module_name = config["analysis"]["workflows"][workflow]["analysis_engine"]
        except KeyError:
            error_msg = ("No analysis engine for workflow \"{}\" specified "
                         "in configuration file. Skipping this workflow "
                         "for project {}".format(workflow, project))
            LOG.error(error_msg)
            raise RuntimeError(error_msg)
        # Import the adapter module specified in the config file (e.g. piper_ngi)
        try:
            analysis_module = importlib.import_module(analysis_engine_module_name)
        except ImportError as e:
            error_msg = ('Skipping project "{}" workflow "{}": couldn\'t import '
                         'module "{}": {}'.format(project, workflow, analysis_engine_module_name, e))
            LOG.error(error_msg)
            # Next project
            continue

        # This is weird
        objects_to_process = []
        if level == "sample":
            for sample in project:
                objects_to_process.append({"project": project, "sample": sample})
        elif level == "seqrun":
            for sample in project:
                for libprep in sample:
                    for seqrun in libprep:
                        objects_to_process.append({"project": project,
                                                   "sample": sample,
                                                   "libprep": libprep,
                                                   "seqrun": seqrun})
        # Still weird and not so great
        for obj_dict in objects_to_process:
            project = obj_dict.get("project")
            sample = obj_dict.get("sample")
            libprep = obj_dict.get("libprep")
            seqrun = obj_dict.get("seqrun")

            try:
                if level == "seqrun":
                    charon_reported_status = charon_session.seqrun_get(project.project_id,
                                                                       sample, libprep,
                                                                       seqrun)['alignment_status']
                else: # sample-level
                    charon_reported_status = charon_session.sample_get(project.project_id,
                                                                       sample)['status']
            except (CharonError, KeyError) as e:
                LOG.warn('Unable to get required information from Charon for '
                          'sample "{}" / project "{}" -- forcing it to new: {}'.format(sample, project, e))
                if level == "seqrun":
                    charon_session.seqrun_update(project.project_id, sample.name, libprep.name, seqrun.name, alignment_status="NEW")
                    charon_reported_status = charon_session.seqrun_get(project.project_id,
                                                                       sample, libprep,
                                                                       seqrun)['alignment_status']
                else:
                    charon_session.sample_update(project.project_id, sample.name, status="NEW")
                    charon_reported_status = charon_session.sample_get(project.project_id,
                                                                       sample)['status']

            # Check Charon to ensure this hasn't already been processed
            if charon_reported_status in ("RUNNING", "DONE"):
                if level == "seqrun":
                    LOG.info('Charon reports seqrun analysis for project "{}" / sample "{}" '
                             '/ libprep "{}" / seqrun "{}" does not need processing '
                             ' (already "{}")'.format(project, sample, libprep, seqrun,
                                                      charon_reported_status))
                else: # Sample
                    LOG.info('Charon reports seqrun analysis for project "{}" / sample "{}" '
                             'does not need processing '
                             ' (already "{}")'.format(project, sample, charon_reported_status))
                continue
            elif charon_reported_status == "FAILED":
                if not restart_failed_jobs:
                    if level == "seqrun":
                        LOG.error('FAILED:  Project "{}" / sample "{}" / library "{}" '
                                  '/ flowcell "{}": Charon reports FAILURE, manual '
                                  'investigation needed!'.format(project, sample, libprep, seqrun))
                    else: # Sample
                        LOG.error('FAILED:  Project "{}" / sample "{}" Charon reports FAILURE, manual '
#.........这里部分代码省略.........