Python domain.Domain类代码示例

 def __init__(self, *args, **kwargs):
     #populate date choices
     self.domain = ""
     allow_adding = True
     if 'domain' in kwargs:
         self.domain = kwargs['domain']
         del kwargs['domain'] #keep the default __init__ from erroring out
     if 'allow_adding' in kwargs:
         allow_adding = kwargs['allow_adding']
         del kwargs['allow_adding']
     super(UpdateWhoisForm, self).__init__(*args, **kwargs)
     if self.domain:
         if allow_adding:
             date_choices = [("","Add New")]
         else:
             date_choices = []
         dmain = Domain.objects(domain=self.domain).first()
         if dmain:
             whois = dmain.whois
             whois.sort(key=lambda w: w['date'], reverse=True)
             for w in dmain.whois:
                 date = datetime.strftime(w['date'],
                                          settings.PY_DATETIME_FORMAT)
                 date_choices.append((date,date))
         self.fields['date'].choices = date_choices

def add_whois(domain, data, date, analyst, editable):
    """
    Add whois information to a domain.

    :param domain: The domain for the whois entry.
    :type domain: str
    :param data: The whois data.
    :type data: str
    :param date: The date for the whois data.
    :type date: datetime.datetime
    :param analyst: The user editing the domain name.
    :type analyst: str
    :param editable: If this entry can be modified.
    :type editable: boolean
    :returns: dict with keys:
              "success" (boolean),
              "whois" (str) if successful,
              "message" (str) if failed.
    """

    domain = Domain.objects(domain=domain).first()
    if not domain:
        return {'success': False,
                'message': "No matching domain found."}
    try:
        whois_entry = domain.add_whois(data, analyst, date, editable)
        domain.save(username=analyst)
        return {"success": True, 'whois': whois_entry}
    except ValidationError, e:
        return {"success": False, "message": e}

def delete_whois(domain, date, analyst):
    """
    Remove whois information for a domain.

    :param domain: The domain for the whois entry.
    :type domain: str
    :param date: The date for the whois data.
    :type date: datetime.datetime
    :param analyst: The user editing the domain name.
    :type analyst: str
    :returns: dict with keys:
              "success" (boolean),
              "message" (str) if failed.
    """

    domain = Domain.objects(domain=domain).first()
    if not domain:
        return {'success': False,
                'message': "No matching domain found."}
    try:
        domain.delete_whois(date)
        domain.save(username=analyst)
        return {'success': True}
    except ValidationError, e:
        return {'success': False, 'message': e}

def edit_domain_name(domain, new_domain, analyst):
    """
    Edit domain name for an entry.

    :param domain: The domain name to edit.
    :type domain: str
    :param new_domain: The new domain name.
    :type new_domain: str
    :param analyst: The user editing the domain name.
    :type analyst: str
    :returns: boolean
    """

    # validate new domain
    (root, validated_domain, error) = get_valid_root_domain(new_domain)
    if error:
        return False

    domain = Domain.objects(domain=domain).first()
    if not domain:
        return False
    try:
        domain.domain = validated_domain
        domain.save(username=analyst)
        return True
    except ValidationError:
        return False

    def create_domain_context(self, identifier, username):
        domain = Domain.objects(id=identifier).first()
        if not domain:
            raise ValueError("Domain not found in database")

        return DomainContext(username=username,
                             _id=identifier,
                             domain_dict=domain.to_dict())

 def clean_date(self):
     date = self.cleaned_data['date']
     if date:
         date_obj = datetime.strptime(self.cleaned_data['date'],
                                      settings.PY_DATETIME_FORMAT)
         domain = Domain.objects(domain=self.domain,
                                 whois__date=date_obj).first()
         if not domain:
             raise forms.ValidationError(u'%s is not a valid date.' % date)

def class_from_value(type_, value):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param value: The value to search for.
    :type value: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if type_ == 'Campaign':
        return Campaign.objects(name=value).first()
    elif type_ == 'Certificate':
        return Certificate.objects(md5=value).first()
    elif type_ == 'Comment':
        return Comment.objects(id=value).first()
    elif type_ == 'Domain':
        return Domain.objects(domain=value).first()
    elif type_ == 'Email':
        return Email.objects(id=value).first()
    elif type_ == 'Event':
        return Event.objects(id=value).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=value).first()
    elif type_ == 'IP':
        return IP.objects(ip=value).first()
    elif type_ == 'PCAP':
        return PCAP.objects(md5=value).first()
    elif type_ == 'RawData':
        return RawData.objects(md5=value).first()
    elif type_ == 'Sample':
        return Sample.objects(md5=value).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=value).first()
    elif type_ == 'Target':
        return Target.objects(email_address=value).first()
    else:
        return None

def process_bulk_add_domain(request, formdict):
    """
    Performs the bulk add of domains by parsing the request data. Batches
    some data into a cache object for performance by reducing large
    amounts of single database queries.

    :param request: Django request.
    :type request: :class:`django.http.HttpRequest`
    :param formdict: The form representing the bulk uploaded data.
    :type formdict: dict
    :returns: :class:`django.http.HttpResponse`
    """

    domain_names = []
    ip_addresses = []
    cached_domain_results = {}
    cached_ip_results = {}

    cleanedRowsData = convert_handsontable_to_rows(request)
    for rowData in cleanedRowsData:
        if rowData != None:
            if rowData.get(form_consts.Domain.DOMAIN_NAME) != None:
                domain = rowData.get(form_consts.Domain.DOMAIN_NAME).strip().lower()
                (root_domain, full_domain, error) = get_valid_root_domain(domain)
                domain_names.append(full_domain)

                if domain != root_domain:
                    domain_names.append(root_domain)

            if rowData.get(form_consts.Domain.IP_ADDRESS) != None:
                ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS)
                ip_type = rowData.get(form_consts.Domain.IP_TYPE)
                (ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type)
                ip_addresses.append(ip_addr)

    domain_results = Domain.objects(domain__in=domain_names)

    ip_results = IP.objects(ip__in=ip_addresses)

    for domain_result in domain_results:
        cached_domain_results[domain_result.domain] = domain_result

    for ip_result in ip_results:
        cached_ip_results[ip_result.ip] = ip_result

    cache = {
        form_consts.Domain.CACHED_RESULTS: cached_domain_results,
        form_consts.IP.CACHED_RESULTS: cached_ip_results,
        "cleaned_rows_data": cleanedRowsData,
    }

    response = parse_bulk_upload(request, parse_row_to_bound_domain_form, add_new_domain_via_bulk, formdict, cache)

    return response

def whois_diff(domain, from_date, to_date):
    """
    Diff whois entries.

    :param domain: The domain to get whois information for.
    :type domain: str
    :param from_date: The date for the first whois entry.
    :type from_date: datetime.datetime
    :param to_date: The date for the second whois entry.
    :type to_date: datetime.datetime
    :returns: dict if failed, str on success.
    """

    domain = Domain.objects(domain=domain).first()
    if not domain:
        return {"success": False, "message": "No matching domain found."}
    return domain.whois_diff(from_date, to_date)

def retrieve_domain(domain, cache):
    """
    Retrieves a domain by checking cache first. If not in cache
    then queries mongo for the domain.

    :param domain: The domain name.
    :type domain: str
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: :class:`crits.domains.domain.Domain`
    """
    domain_obj = None
    cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)

    if cached_results != None:
        domain_obj = cached_results.get(domain.lower())
    else:
        domain_obj = Domain.objects(domain_iexact=domain).first()

    return domain_obj

    def _delete_all_analysis_results(self, md5_digest, service_name):
        """
        Delete all analysis results for this service.
        """

        obj = Sample.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = PCAP.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Certificate.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = RawData.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Event.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Indicator.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Domain.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = IP.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()

def get_domain_details(domain, analyst):
    """
    Generate the data to render the Domain details template.

    :param domain: The name of the Domain to get details for.
    :type domain: str
    :param analyst: The user requesting this information.
    :type analyst: str
    :returns: template (str), arguments (dict)
    """

    template = None
    allowed_sources = user_sources(analyst)
    dmain = Domain.objects(domain=domain, source__name__in=allowed_sources).first()
    if not dmain:
        error = "Either no data exists for this domain" " or you do not have permission to view it."
        template = "error.html"
        args = {"error": error}
        return template, args

    forms = {}
    # populate whois data into whois form
    # and create data object (keyed on date) for updating form on date select
    whois_data = {"": ""}  # blank info for "Add New" option
    initial_data = {"data": " "}
    raw_data = {}
    whois = getattr(dmain, "whois", None)
    if whois:
        for w in whois:
            # build data as a display-friendly string
            w.date = datetime.datetime.strftime(w.date, settings.PY_DATETIME_FORMAT)
            from whois_parser import WhoisEntry

            # prettify the whois data
            w.data = unicode(WhoisEntry.from_dict(w.data))
            if "text" not in w:  # whois data was added with old data format
                w.text = w.data
            # also save our text blob for easy viewing of the original data
            whois_data[w.date] = (w.data, w.text)
        # show most recent entry first
        initial_data = {"data": whois[-1].data, "date": whois[-1].date}
        raw_data = {"data": whois[-1].text, "date": whois[-1].date}

    whois_len = len(whois_data) - 1  # subtract one to account for blank "Add New" entry
    whois_data = json.dumps(whois_data)

    dmain.sanitize_sources(username="%s" % analyst, sources=allowed_sources)

    forms["whois"] = UpdateWhoisForm(initial_data, domain=domain)
    forms["raw_whois"] = UpdateWhoisForm(raw_data, domain=domain, allow_adding=False)
    forms["diff_whois"] = DiffWhoisForm(domain=domain)

    # remove pending notifications for user
    remove_user_from_notification("%s" % analyst, dmain.id, "Domain")

    # subscription
    subscription = {
        "type": "Domain",
        "id": dmain.id,
        "subscribed": is_user_subscribed("%s" % analyst, "Domain", dmain.id),
    }

    # objects
    objects = dmain.sort_objects()

    # relationships
    relationships = dmain.sort_relationships("%s" % analyst, meta=True)

    # relationship
    relationship = {"type": "Domain", "value": dmain.id}

    # comments
    comments = {"comments": dmain.get_comments(), "url_key": dmain.domain}

    # screenshots
    screenshots = dmain.get_screenshots(analyst)

    # favorites
    favorite = is_user_favorite("%s" % analyst, "Domain", dmain.id)

    # services
    service_list = get_supported_services("Domain")

    # analysis results
    service_results = dmain.get_analysis_results()

    args = {
        "objects": objects,
        "relationships": relationships,
        "comments": comments,
        "favorite": favorite,
        "relationship": relationship,
        "subscription": subscription,
        "screenshots": screenshots,
        "domain": dmain,
        "forms": forms,
        "whois_data": whois_data,
        "service_list": service_list,
        "service_results": service_results,
        "whois_len": whois_len,
#.........这里部分代码省略.........

def upsert_domain(domain, source, username=None, campaign=None,
                  confidence=None, bucket_list=None, ticket=None, cache={}):
    """
    Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
    dictionary objects.

    :param domain: The domain to add/update.
    :type domain: str
    :param source: The name of the source.
    :type source: str
    :param username: The user adding/updating the domain.
    :type username: str
    :param campaign: The campaign to attribute to this domain.
    :type campaign: list, str
    :param confidence: Confidence for the campaign attribution.
    :type confidence: str
    :param bucket_list: List of buckets to add to this domain.
    :type bucket_list: list, str
    :param ticket: The ticket for this domain.
    :type ticket: str
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "object" the domain that was added,
              "is_domain_new" (boolean)
    """

    # validate domain and grab root domain
    (root, domain, error) = get_valid_root_domain(domain)
    if error:
        return {'success': False, 'message': error}

    is_fqdn_domain_new = False
    is_root_domain_new = False

    if not campaign:
        campaign = []
    # assume it's a list, but check if it's a string
    elif isinstance(campaign, basestring):
        c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
        campaign = [c]

    # assume it's a list, but check if it's a string
    if isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = ''
        instance.method = ''
        instance.analyst = username
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        source = [s]

    fqdn_domain = None
    root_domain = None
    cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)

    if cached_results != None:
        if domain != root:
            fqdn_domain = cached_results.get(domain)
            root_domain = cached_results.get(root)
        else:
            root_domain = cached_results.get(root)
    else:
        #first find the domain(s) if it/they already exist
        root_domain = Domain.objects(domain=root).first()
        if domain != root:
            fqdn_domain = Domain.objects(domain=domain).first()

    #if they don't exist, create them
    if not root_domain:
        root_domain = Domain()
        root_domain.domain = root
        root_domain.source = []
        root_domain.record_type = 'A'
        is_root_domain_new = True

        if cached_results != None:
            cached_results[root] = root_domain
    if domain != root and not fqdn_domain:
        fqdn_domain = Domain()
        fqdn_domain.domain = domain
        fqdn_domain.source = []
        fqdn_domain.record_type = 'A'
        is_fqdn_domain_new = True

        if cached_results != None:
            cached_results[domain] = fqdn_domain

    # if new or found, append the new source(s)
    for s in source:
        if root_domain:
            root_domain.add_source(s)
        if fqdn_domain:
            fqdn_domain.add_source(s)

    #campaigns
#.........这里部分代码省略.........

def class_from_id(type_, _id):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param _id: The ObjectId to search for.
    :type _id: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.core.source_access import SourceAccess
    from crits.core.user_role import UserRole
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator, IndicatorAction
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData, RawDataType
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if not _id:
        return None

    # make sure it's a string
    _id = str(_id)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if not ObjectId.is_valid(_id.decode('utf8')):
        return None

    if type_ == 'Actor':
        return Actor.objects(id=_id).first()
    elif type_ == 'Backdoor':
        return Backdoor.objects(id=_id).first()
    elif type_ == 'ActorThreatIdentifier':
        return ActorThreatIdentifier.objects(id=_id).first()
    elif type_ == 'Campaign':
        return Campaign.objects(id=_id).first()
    elif type_ == 'Certificate':
        return Certificate.objects(id=_id).first()
    elif type_ == 'Comment':
        return Comment.objects(id=_id).first()
    elif type_ == 'Domain':
        return Domain.objects(id=_id).first()
    elif type_ == 'Email':
        return Email.objects(id=_id).first()
    elif type_ == 'Event':
        return Event.objects(id=_id).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=_id).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=_id).first()
    elif type_ == 'IndicatorAction':
        return IndicatorAction.objects(id=_id).first()
    elif type_ == 'IP':
        return IP.objects(id=_id).first()
    elif type_ == 'PCAP':
        return PCAP.objects(id=_id).first()
    elif type_ == 'RawData':
        return RawData.objects(id=_id).first()
    elif type_ == 'RawDataType':
        return RawDataType.objects(id=_id).first()
    elif type_ == 'Sample':
        return Sample.objects(id=_id).first()
    elif type_ == 'SourceAccess':
        return SourceAccess.objects(id=_id).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=_id).first()
    elif type_ == 'Target':
        return Target.objects(id=_id).first()
    elif type_ == 'UserRole':
        return UserRole.objects(id=_id).first()
    else:
        return None

#.........这里部分代码省略.........

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.value
        url_contains_ip = False
        if ind_type in ("URI - Domain Name", "URI - URL"):
            if ind_type == "URI - URL":
                domain_or_ip = urlparse.urlparse(ind_value).hostname
            elif ind_type == "URI - Domain Name":
                domain_or_ip = ind_value
            (sdomain, fqdn) = get_domain(domain_or_ip)
            if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(sdomain, fqdn, indicator.source,
                                            '%s' % analyst, None,
                                            bucket_list=bucket_list, cache=cache)
                    if not success['success']:
                        return {'success': False, 'message': success['message']}

                if not success or not 'object' in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success['object']

        if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = 'Address - ipv4-addr'
                except DjangoValidationError: