Python observables.Observable类代码示例

    def match(self):
        """Match observables against Yeti's intelligence repository.

        Takes an array of observables, expands them and tries to match them against specific indicators or known observables.

        To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
        the "expanded" observable array will also include the hostname ``google.com``.

        :<json [string] observables: An array of observables to be analyzed

        :>json [Entity] entities: Related ``Entity`` objects
        :>json [Observable] known: ``Observable`` objects that are already present in database
        :>json [Indicator] matches: ``Indicators`` that matched observables
        :>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
        :>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
        """

        params = request.json
        observables = params.pop('observables', [])
        fetch_neighbors = params.pop('fetch_neighbors', True)
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown and current_user.has_permission('observable', 'write'):
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)

        return render(data)

def derive(observables):
    """Indicate that the module needs a specific attribute to work properly.

    This function is only useful in abstract modules, in order to make sure
    that modules that inherit from this class correctly defines needed class
    attributes.

    Args:
        variables: a string or an array of strings containing the name of
            needed class attributes.

    Raises:
        ModuleInitializationError: One of the needed attributes is not
            correctly defined.
    """

    new = []
    observables = list(iterify(observables))
    for i, observable in enumerate(observables):
        try:
            t = Observable.guess_type(observable)
            temp = t(value=observable)
            temp.clean()
            observable = temp.value
            observables[i] = observable
            for a in analyzers.get(t, []):
                new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return observables + derive(new)

 def analyze_outdated(self):
     # do outdated logic
     fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
     if self.EXPIRATION:
         fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION})
     fltr &= Q(**self.CUSTOM_FILTER) & Q(_cls__contains=self.ACTS_ON)
     self.bulk(Observable.objects(fltr))

def derive(strings):
    values = set()
    observables = set()

    for string in iterify(strings):
        if string:
            try:
                t = Observable.guess_type(string)
                observable = t(value=string)
                observable.normalize()
                observables.add(observable)
                values.add(observable.value)
            except ObservableValidationError:
                values.add(string)

    new = []
    for observable in observables:
        for a in analyzers.get(observable.__class__, []):
            new.extend([
                n for n in a.analyze_string(observable.value)
                if n and n not in values
            ])

    if len(new) == 0:
        return values, values
    else:
        _, extended = derive(new + list(values))
        return values, extended

    def analyze(observable, results):
        links = set()

        params = {'query': observable.value}

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(
                record['firstSeen'], "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(
                record['lastSeen'], "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))
            else:
                links.update(
                    new.link_to(
                        observable, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))

        return list(links)

    def post(self):
        q = request.get_json(silent=True)
        params = q.pop("params", {})
        observables = []

        for o in q["observables"]:
            try:
                obs = Observable.guess_type(o['value'])(value=o['value'])
                obs.clean()
                observables.append(obs.value)

                # Save observables & eventual tags to database
                if params.get('save_query', False):
                    obs = obs.save()
                    obs.tag(o.get("tags", []))
                    obs.add_source("query")
            except ObservableValidationError:
                continue

        # match observables with known indicators
        data = match_observables([o for o in observables])

        # find related observables (eg. URLs for domain, etc.)
        # related_observables = [obs.get_related() for obs in observables]
        # data = self.match_observables(related_observable)
        #
        # we need to find a way to degrade the "confidence" in
        # hits obtained from related observables

        return render(data, "analysis.html")

    def analyze(self, dict):
        observable = dict['title']
        description = dict['description'].lower()
        context = {}
        context['description'] = "{} C2 server".format(description)
        context['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context['source'] = self.name

        try:
            e = Observable.add_text(observable)
        except ObservableValidationError as e:
            logging.error(e)
            return

        e.add_context(context)
        e.add_source("feed")

        tags = ['malware', 'c2', description, 'crimeware']
        if description == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif description == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif description in ['zeus', 'citadel']:
            tags.extend(['banker'])

        e.tag(tags)

 def execute(self):
     self.export_file_handle = codecs.open(self.output_file, 'w+', "utf-8")
     q = Q(tags__name__in=[t.name for t in self.include_tags]) & Q(tags__name__nin=[t.name for t in self.exclude_tags])
     q &= Q(_cls__contains=self.acts_on)
     output = self.template.render(Observable.objects(q))
     self.write(output)
     self.export_file_handle.close()

 def each(url):
     try:
         host = ProcessUrl.analyze_string(url.value)[0]
         h = Observable.guess_type(host).get_or_create(value=host)
         h.add_source("analytics")
         Link.connect(src=url, dst=h)
     except ObservableValidationError:
         logging.error("An error occurred when trying to add {} to the database".format(host))

 def each(url):
     try:
         host = ProcessUrl.analyze_string(url.value)[0]
         h = Observable.guess_type(host).get_or_create(value=host)
         h.add_source("analytics")
         url.active_link_to(h, "hostname", "ProcessUrl", clean_old=False)
         return h
     except ObservableValidationError:
         logging.error("An error occurred when trying to add {} to the database".format(host))

 def enrich(self):
     return "ENRICH"
     if request.method == "POST":
         lines = request.form['bulk-text'].split('\n')
         for l in lines:
             obs = refang(l.split(',')[0])
             tags = refang(l.split(',')[1:])
             o = Observable.add_text(obs)
             o.tag(tags)
     return render_template('observable/query.html')

def match_observables(observables):
    # Remove empty observables
    observables = [observable for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))
    added_entities = set()

    data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}

    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        for link, node in (o.incoming()):
            if isinstance(node, Observable):
                if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                    data['neighbors'].append((link.info(), node.info()))

    for o, i in Indicator.search(extended_query):
        o = Observable.add_text(o)
        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors().values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                if node.name not in added_entities:
                    nodeinfo = node.info()
                    nodeinfo['type'] = node.type
                    data["entities"].append(nodeinfo)
                    added_entities.add(node.name)

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)
        del_from_set(data["unknown"], o.value)

    return data

    def analyze_outdated(self):
        class_filter = Q()
        for acts_on in iterify(self.ACTS_ON):
            class_filter |= Q(_cls="Observable.{}".format(acts_on))

        # do outdated logic
        fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
        if self.EXPIRATION:
            fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION})
        fltr &= self.CUSTOM_FILTER & class_filter
        self.bulk(Observable.objects(fltr).no_cache())

    def post(self, action=None):
        if action == 'merge':
            tags = request.json['merge']
            merge_into = Tag.objects.get(name=request.json['merge_into'])
            make_dict = request.json['make_dict']

            merged = 0
            for tag in tags:
                Observable.change_all_tags(tags, merge_into.name)
                oldtag = Tag.objects.get(name=tag)
                merge_into.count += oldtag.count
                merge_into.produces += [i for i in oldtag.produces if i not in merge_into.produces and i != merge_into]
                merge_into.save()
                oldtag.delete()
                merged += 1

            if make_dict:
                merge_into.add_replaces(tags)

            return render({"merged": merged, "into": merge_into.name})

    def execute(self):
        q_include = Q()
        for t in self.include_tags:
            q_include |= Q(tags__match={"name": t.name, "fresh": True})
        q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
        q = (
            Q(tags__not__size=0, tags__match={"fresh": True})
            & q_include
            & q_exclude
            & Q(_cls="Observable.{}".format(self.acts_on))
        )

        return self.template.render(self.filter_ignore_tags(Observable.objects(q).no_cache()), self.output_file)