Python mall.get_mall函数代码示例

def load_variants_for_cohort_list(project, cohorts):

    for cohort in cohorts:
        family_list = []
        print "Adding {}".format(cohort.cohort_id)
        family_list.append({
            'project_id': cohort.project.project_id,
            'family_id': cohort.cohort_id,
            'individuals': cohort.indiv_id_list(),
            })

        # add all families from this vcf to the datastore
        get_mall(project.project_id).variant_store.add_family_set(family_list)

        vcf_files = cohort.get_vcf_files()

        # create the VCF ID map
        vcf_id_map = {}
        for individual in cohort.get_individuals():
            if individual.vcf_id:
                vcf_id_map[individual.vcf_id] = individual.indiv_id

        # load them all into the datastore
        for vcf_file in vcf_files:
            family_tuple_list = [(f['project_id'], f['family_id']) for f in family_list]
            get_mall(project.project_id).variant_store.load_family_set(
                vcf_file.path(),
                family_tuple_list,
                reference_populations=project.get_reference_population_slugs(),
                vcf_id_map=vcf_id_map,
            )

def mendelian_variant_search_spec(request):

    project, family = get_project_and_family_for_user(request.user, request.GET)

    # TODO: use form

    search_hash = request.GET.get('search_hash')
    search_spec_dict, variants = cache_utils.get_cached_results(project.project_id, search_hash)
    search_spec = MendelianVariantSearchSpec.fromJSON(search_spec_dict)
    if variants is None:
        variants = api_utils.calculate_mendelian_variant_search(search_spec, family.xfamily())
    else:
        variants = [Variant.fromJSON(v) for v in variants]
    add_extra_info_to_variants_family(get_reference(), family, variants)
    return_type = request.GET.get('return_type')
    if return_type == 'json' or not return_type:
        return JSONResponse({
            'is_error': False,
            'variants': [v.toJSON() for v in variants],
            'search_spec': search_spec_dict,
        })
    elif request.GET.get('return_type') == 'csv':
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="results_{}.csv"'.format(search_hash)
        writer = csv.writer(response)
        indiv_ids = family.indiv_ids_with_variant_data()
        headers = xbrowse_displays.get_variant_display_headers(get_mall(project.project_id), project, indiv_ids)
        writer.writerow(headers)
        for variant in variants:
            fields = xbrowse_displays.get_display_fields_for_variant(get_mall(project.project_id), project, variant, indiv_ids)
            writer.writerow(fields)
        return response

def load_variants_for_family_list(project, families, vcf_file, mark_as_loaded=True):
    """
    Reload variants for a list of families, all from the same vcf
    """
    family_list = []
    for family in families:
        family_list.append({
            'project_id': family.project.project_id,
            'family_id': family.family_id,
            'individuals': family.indiv_ids_with_variant_data(),
        })

    # add all families from this vcf to the datastore
    get_mall(project.project_id).variant_store.add_family_set(family_list)

    # create the VCF ID map
    vcf_id_map = {}
    for family in families:
        for individual in family.get_individuals():
            if individual.vcf_id:
                vcf_id_map[individual.vcf_id] = individual.indiv_id

    # load them all into the datastore
    family_tuple_list = [(f['project_id'], f['family_id']) for f in family_list]
    get_mall(project.project_id).variant_store.load_family_set(
        vcf_file,
        family_tuple_list,
        reference_populations=project.get_reference_population_slugs(),
        vcf_id_map=vcf_id_map,
        mark_as_loaded=mark_as_loaded,
    )

    # finish up each family
    for family in families:
        _family_postprocessing(family)

 def handle(self, *args, **options):
     for project_id in args:
         print("Deleting data from mongodb for project: " + project_id)
         p = Project.objects.get(project_id = project_id)
         get_mall(p).variant_store.delete_project(project_id)
         get_project_datastore(p).delete_project_store(project_id)
         print("Done")

def delete_project(project_id):
    """
    Delete a project and perform any cleanup (ie. deleting from datastore and removing temp files)
    """
    project = Project.objects.get(project_id=project_id)
    get_mall(project_id).variant_store.delete_project(project_id)
    project.individual_set.all().delete()
    project.family_set.all().delete()
    project.delete()

    def handle(self, *args, **options):
        number_of_variants_to_check = int(options.get("number_of_variants_to_check") or 20000)

        if not args:
            args = [p.project_id for p in Project.objects.all()]
            args.reverse()

        for project_id in args:
            try:
                project = Project.objects.get(project_id=project_id)
            except:
                print("ERROR: Project not found. Skipping..")
                continue
            all_counter = 0
            #found_counter = 0
            not_found_counter = 0
            not_found_variants = []
            for vcf_file in project.get_all_vcf_files():
                path = vcf_file.file_path
                #print("Processing %s - %s" % (project.project_id, path))
                if not os.path.isfile(path) and path.endswith(".vcf"):
                    path = path + ".gz"
                if path.endswith(".gz"):
                    f = gzip.open(path)
                else:
                    f = open(path)
                if f:
                    for variant in vcf_stuff.iterate_vcf(f):
                        all_counter += 1
                        try:
                            get_mall(project_id).annotator.get_annotation(variant.xpos, variant.ref, variant.alt)
                        except ValueError, e:
                            not_found_counter += 1
                            if len(not_found_variants) < 30:
                                chrom, pos = genomeloc.get_chr_pos(variant.xpos)
                                chrom = chrom.replace("chr","")
                                ref, alt = variant.ref, variant.alt
                                not_found_variants.append("%(chrom)s-%(pos)s-%(ref)s-%(alt)s" % locals())
                            #print("WARNING: variant not found in annotator cache: " + str(e))
                            #if not_found_counter > 5:
                            #    print("---- ERROR: 5 variants not found. Project %s should be reloaded." % project_id)
                            #    break
                            found_counter = 0
                        #else:
                        #    found_counter += 1
                        #    if found_counter > 15000:
                        #        #print("---- Found 5000 variants in a row. Project %s looks ok." % project_id)
                        #        break
                        if all_counter >= number_of_variants_to_check:
                            fraction_missing = float(not_found_counter) / all_counter
                            if not_found_counter > 10:
                                print("---- ERROR: (%(fraction_missing)0.2f%%)  %(not_found_counter)s / %(all_counter)s variants not found. Project %(project_id)s should be reloaded. Examples: " % locals())

                                for v in not_found_variants:
                                    print("http://exac.broadinstitute.org/variant/" + v)
                            break

def delete_family(project_id, family_id):
    """
    Delete a project and perform any cleanup (ie. deleting from datastore and removing temp files)
    """
    family = Family.objects.get(project__project_id=project_id, family_id=family_id)
    for individual in family.get_individuals():
        individual.family = None
        individual.save()
    get_mall(project_id).variant_store.delete_family(project_id, family_id)
    family.delete()

def delete_family(project_id, family_id):
    """
    Delete a project and perform any cleanup (ie. deleting from datastore and removing temp files)
    """
    family = Family.objects.get(project__project_id=project_id, family_id=family_id)
    for individual in family.get_individuals():
        update_xbrowse_model(individual, family=None)

    get_mall(project_id).variant_store.delete_family(project_id, family_id)
    delete_xbrowse_model(family)

def load_project_variants(project_id, force_load_annotations=False, force_load_variants=False, ignore_csq_in_vcf=False, start_from_chrom=None, end_with_chrom=None):
    """
    Load any families and cohorts in this project that aren't loaded already
    """
    print "Loading project %s" % project_id
    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- loading project: " + project_id + " - db.variants cache"))
    project = Project.objects.get(project_id=project_id)

    for vcf_obj in sorted(project.get_all_vcf_files(), key=lambda v:v.path()):
        if not os.path.isfile(vcf_obj.path()):
            print("Skipping " + vcf_obj.path())
            continue

        r = vcf.VCFReader(filename=vcf_obj.path())
        if not ignore_csq_in_vcf and "CSQ" not in r.infos:
            raise ValueError("VEP annotations not found in VCF: " + vcf_obj.path())

        mall.get_annotator().add_preannotated_vcf_file(vcf_obj.path(), force=force_load_annotations, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)


    # batch load families by VCF file
    for vcf_file, families in project.families_by_vcf().items():
        if not force_load_variants:
            # filter out families that have already finished loading
            families = [f for f in families if get_mall(project).variant_store.get_family_status(project_id, f.family_id) != 'loaded']

        for i in xrange(0, len(families), settings.FAMILY_LOAD_BATCH_SIZE):
            load_variants_for_family_list(project, families[i:i+settings.FAMILY_LOAD_BATCH_SIZE], vcf_file, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)

    # now load cohorts
    load_cohorts(project_id)

def load_project_variants(project_id, force_annotations=False, ignore_csq_in_vcf=False):
    """
    Load any families and cohorts in this project that aren't loaded already 
    """
    print "Loading project %s" % project_id
    print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- loading project: " + project_id + " - db.variants cache"))
    project = Project.objects.get(project_id=project_id)

    for vcf_obj in project.get_all_vcf_files():
        r = vcf.VCFReader(filename=vcf_obj.path())
        if not ignore_csq_in_vcf and "CSQ" not in r.infos:
            raise ValueError("VEP annotations not found in VCF: " + vcf_file)

        mall.get_annotator().add_preannotated_vcf_file(vcf_obj.path(), force=force_annotations)
        

    # batch load families by VCF file
    for vcf_file, families in project.families_by_vcf().items():
        families = [f for f in families if get_mall(project.project_id).variant_store.get_family_status(project_id, f.family_id) != 'loaded']
        for i in xrange(0, len(families), settings.FAMILY_LOAD_BATCH_SIZE):
            print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S  -- loading project: " + project_id + " - families batch %d - %d families" % (i, len(families[i:i+settings.FAMILY_LOAD_BATCH_SIZE])) ))
            load_variants_for_family_list(project, families[i:i+settings.FAMILY_LOAD_BATCH_SIZE], vcf_file)

    # now load cohorts
    load_cohorts(project_id)

def get_variants_for_inheritance_for_project(project, inheritance_mode):
    """
    Get the variants for this project / inheritance combo
    Return dict of family -> list of variants
    """

    # create search specification
    # this could theoretically differ by project, if there are different reference populations
    variant_filter = get_default_variant_filter("moderate_impact")
    variant_filter.ref_freqs.append(("1kg_wgs_phase3", g1k_freq_threshold))
    variant_filter.ref_freqs.append(("1kg_wgs_phase3_popmax", g1k_popmax_freq_threshold))
    variant_filter.ref_freqs.append(("exac_v3", exac_freq_threshold))
    variant_filter.ref_freqs.append(("exac_v3_popmax", exac_popmax_threshold))
    quality_filter = {"vcf_filter": "pass", "min_gq": GQ_threshold, "min_ab": AB_threshold}

    # run MendelianVariantSearch for each family, collect results
    families = project.get_families()
    for i, family in enumerate(families):
        sys.stdout.write(
            "Processing %s - family %s  (%d / %d) .." % (inheritance_mode, family.family_id, i + 1, len(families))
        )
        variant_list = list(
            get_variants_with_inheritance_mode(
                get_mall(project.project_id),
                family.xfamily(),
                inheritance_mode,
                variant_filter=variant_filter,
                quality_filter=quality_filter,
            )
        )
        yield family, variant_list
        print(" got %d variants" % len(variant_list))

def combine_mendelian_families_variants(request):

    project, family_group = utils.get_project_and_family_group_for_user(request.user, request.GET)

    form = api_forms.CombineMendelianFamiliesVariantsForm(request.GET)
    if form.is_valid():
        variants_grouped = get_variants_by_family_for_gene(
            get_mall(project.project_id),
            [f.xfamily() for f in form.cleaned_data['families']],
            form.cleaned_data['inheritance_mode'],
            form.cleaned_data['gene_id'],
            variant_filter=form.cleaned_data['variant_filter'],
            quality_filter=form.cleaned_data['quality_filter']
        )
        variants_by_family = []
        for family in form.cleaned_data['families']:
            variants = variants_grouped[(family.project.project_id, family.family_id)]
            add_extra_info_to_variants_family(get_reference(), family, variants)
            variants_by_family.append({
                'project_id': family.project.project_id,
                'family_id': family.family_id,
                'family_name': str(family),
                'variants': [v.toJSON() for v in variants],
            })
        return JSONResponse({
            'is_error': False,
            'variants_by_family': variants_by_family,
        })

    else:
        return JSONResponse({
            'is_error': True,
            'error': server_utils.form_error_string(form)
        })

    def handle(self, *args, **options):
        if len(args) < 1:
            print("Please provide the project_id. The individual_id(s) are optional")
            return

        project_id = args[0]

        try:
            project = Project.objects.get(project_id=project_id)
        except ObjectDoesNotExist:
            sys.exit("Invalid project id: " + project_id)

        individual_ids = args[1:]
        try:
            if individual_ids:
                individual_ids = [Individual.objects.get(project=project, indiv_id=individual_id) for individual_id in individual_ids]
            else:
                individual_ids = [i for i in Individual.objects.filter(project=project)]
        except ObjectDoesNotExist:
            sys.exit("Invalid individual ids: " + str(individual_ids))

        for i in individual_ids:
            family_collection = get_mall(project_id).variant_store._get_family_collection(project_id, i.family.family_id)
            if family_collection is None:
                print("WARNING: Family %s data not loaded in variant datastore. Skipping individual %s." % (i.family.family_id, i))
                continue
            self.handle_individual(project, i)
        print("Finished generating report")

def load_cohorts(project_id):
    # now load cohorts
    project = Project.objects.get(project_id=project_id)
    for vcf_file, cohorts in project.cohorts_by_vcf().items():
        cohorts = [c for c in cohorts if get_mall(project).variant_store.get_family_status(project_id, c.cohort_id) != 'loaded']
        for i in xrange(0, len(cohorts), settings.FAMILY_LOAD_BATCH_SIZE):
            print("Loading project %s - cohorts: %s" % (project_id, cohorts[i:i+settings.FAMILY_LOAD_BATCH_SIZE]))
            load_variants_for_cohort_list(project, cohorts[i:i+settings.FAMILY_LOAD_BATCH_SIZE])

    def handle(self, *args, **options):
        project_id = options['project_id']
        print("Loading data into project: " + project_id)
        project = Project.objects.get(project_id = project_id)

        cnv_filename = options['cnv_filename']
        bed_files_directory = options['bed_files_directory']
        
        if not os.path.isfile(cnv_filename):
            raise ValueError("CNV file %s doesn't exist" % options['cnv_filename'])
        
        with open(cnv_filename) as f:
            header_fields = f.readline().rstrip('\n').split('\t')
            for line in f:
                fields = line.rstrip('\n').split('\t')
                row_dict = dict(zip(header_fields, fields))

                chrom = "chr"+row_dict['chr']
                start = int(row_dict['start'])
                end = int(row_dict['end'])
                #left_overhang = int(row_dict['left_overhang_start'])
                #right_overhang = int(row_dict['right_overhang_end'])

                sample_id = row_dict['sample']
                try:
                    i = Individual.objects.get(project=project, indiv_id__istartswith=sample_id)
                except Exception as e:
                    print("WARNING: %s: %s not found in %s" % (e, sample_id, project))
                    continue
                
                bed_file_path = os.path.join(bed_files_directory, "%s.bed" % sample_id)
                if not os.path.isfile(bed_file_path):
                    print("WARNING: .bed file not found: " + bed_file_path)

                    if i.cnv_bed_file != bed_file_path:
                        print("Setting cnv_bed_file path to %s" % bed_file_path)
                        i.cnv_bed_file = bed_file_path
                        i.save()
                
                project_collection = get_project_datastore(project)._get_project_collection(project_id)
                family_collection = get_mall(project).variant_store._get_family_collection(project_id, i.family.family_id)

                for collection in filter(None, [project_collection, family_collection]):
                    
                    collection.update_many(
                        {'$and': [
                            {'xpos': {'$gte': genomeloc.get_single_location(chrom, start)} },
                            {'xpos': {'$lte': genomeloc.get_single_location(chrom, end)}}
                        ]},
                        {'$set': {'genotypes.%s.extras.cnvs' % i.indiv_id: row_dict}})

                    #result = list(collection.find({'$and' : [
                    #       {'xpos': {'$gte':  genomeloc.get_single_location(chrom, start)}},
                    #       {'xpos' :{'$lte': genomeloc.get_single_location(chrom, end)}}]},
                    #   {'genotypes.%s.extras.cnvs' % i.indiv_id :1 }))
                    #print(chrom, start, end, len(result), result[0] if result else None)

        print("Done")