Python Observable.add_text方法代码示例

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def match(self):
        """Match observables against Yeti's intelligence repository.

        Takes an array of observables, expands them and tries to match them against specific indicators or known observables.

        To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
        the "expanded" observable array will also include the hostname ``google.com``.

        :<json [string] observables: An array of observables to be analyzed

        :>json [Entity] entities: Related ``Entity`` objects
        :>json [Observable] known: ``Observable`` objects that are already present in database
        :>json [Indicator] matches: ``Indicators`` that matched observables
        :>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
        :>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
        """

        params = request.json
        observables = params.pop('observables', [])
        fetch_neighbors = params.pop('fetch_neighbors', True)
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown and current_user.has_permission('observable', 'write'):
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)

        return render(data)

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def analyze(self, dict):
        observable = dict['title']
        description = dict['description'].lower()
        context = {}
        context['description'] = "{} C2 server".format(description)
        context['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context['source'] = self.name

        try:
            e = Observable.add_text(observable)
        except ObservableValidationError as e:
            logging.error(e)
            return

        e.add_context(context)
        e.add_source("feed")

        tags = ['malware', 'c2', description, 'crimeware']
        if description == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif description == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif description in ['zeus', 'citadel']:
            tags.extend(['banker'])

        e.tag(tags)

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value}

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(
                record['firstSeen'], "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(
                record['lastSeen'], "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))
            else:
                links.update(
                    new.link_to(
                        observable, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))

        return list(links)

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
 def enrich(self):
     return "ENRICH"
     if request.method == "POST":
         lines = request.form['bulk-text'].split('\n')
         for l in lines:
             obs = refang(l.split(',')[0])
             tags = refang(l.split(',')[1:])
             o = Observable.add_text(obs)
             o.tag(tags)
     return render_template('observable/query.html')

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup('rdata', observable, api_key):
            new = Observable.add_text(record['rrname'])
            new.add_source('analytics')

            links.update(
                new.link_to(
                    observable,
                    source='DNSDB Passive DNS',
                    description='{} record'.format(record['rrtype']),
                    first_seen=record['first_seen'],
                    last_seen=record['last_seen']))

        return list(links)

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
                l = Link.connect(h, e)
                l.add_history(tag=rtype, description='{} record'.format(rtype))
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)
        return generated

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup('rrset', hostname, api_key):
            for observable in record['rdata']:
                observable = Observable.add_text(observable)
                observable.add_source('analytics')

                links.update(
                    hostname.link_to(
                        observable,
                        source='DNSDB Passive DNS',
                        description='{} record'.format(record['rrtype']),
                        first_seen=record['first_seen'],
                        last_seen=record['last_seen']))

        return list(links)

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(
                            hostname,
                            "First seen IP",
                            self.name,
                            clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
def match_observables(observables):
    # Remove empty observables
    observables = [observable for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))
    added_entities = set()

    data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}

    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        for link, node in (o.incoming()):
            if isinstance(node, Observable):
                if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                    data['neighbors'].append((link.info(), node.info()))

    for o, i in Indicator.search(extended_query):
        o = Observable.add_text(o)
        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors().values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                if node.name not in added_entities:
                    nodeinfo = node.info()
                    nodeinfo['type'] = node.type
                    data["entities"].append(nodeinfo)
                    added_entities.add(node.name)

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)
        del_from_set(data["unknown"], o.value)

    return data

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
MalwareFamily("rootkit").save()
MalwareFamily("trojan").save()
MalwareFamily("dropper").save()


t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])

Tag.get_or_create(name="crimeware").add_produces("malware")

Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save()


url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags

# print url.find_tags()

# import pdb; pdb.set_trace()



## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "delivery"
bartalex.tags = ["bartalex"]
bartalex.save()

# 需要导入模块: from core.observables import Observable [as 别名]
# 或者: from core.observables.Observable import add_text [as 别名]
def match_observables(observables, save_matches=False, fetch_neighbors=True):
    # Remove empty observables
    observables = [refang(observable) for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))

    data = {
        "matches": [],
        "unknown": set(observables),
        "entities": {},
        "known": [],
        "neighbors": [],
    }

    # add to "known"
    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        if fetch_neighbors:
            for link, node in (o.incoming()):
                if isinstance(node, Observable):
                    if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                        data['neighbors'].append((link.info(), node.info()))

        for nodes in o.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"observables": []}
                if 'observables' not in ent['matches']:
                    ent['matches']['observables'] = []

                info = node.info()
                o_info = o.info()
                info['matched_observable'] = {
                    "value": o_info['value'],
                    "tags": [t['name'] for t in o_info['tags']],
                    "human_url": o_info['human_url'],
                    "url": o_info['url']
                }
                if info not in ent['matches']['observables']:
                    ent['matches']['observables'].append(info)
                data['entities'][node.name] = ent

    # add to "matches"
    for o, i in Indicator.search(extended_query):
        if save_matches:
            o = Observable.add_text(o)
        else:
            o = Observable.guess_type(o)(value=o)
            try:
                o.validate()
            except ObservableValidationError:
                pass
            try:
                o = Observable.objects.get(value=o.value)
            except Exception:
                pass

        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"indicators": []}
                if 'indicators' not in ent['matches']:
                    ent['matches']['indicators'] = []

                info = i.info()
                info['matched_observable'] = o.value
                if info not in ent['matches']['indicators']:
                    ent['matches']['indicators'].append(info)
                data['entities'][node.name] = ent

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)

    data['entities'] = data['entities'].values()
    return data