Python model.Entity类代码示例

 def setUp(self):
     super(EntitiesTestCase, self).setUp()
     self.rolex = self.create_user(foreign_id='user_3')
     self.col = Collection()
     self.col.label = 'Original Collection'
     self.col.foreign_id = 'test_coll_entities'
     db.session.add(self.col)
     self.col_other = Collection()
     self.col_other.label = 'Other Collection'
     self.col_other.foreign_id = 'test_coll_entities_other'
     db.session.add(self.col_other)
     db.session.flush()
     self.ent = Entity.create({
         'schema': 'LegalEntity',
         'properties': {
             'name': 'Winnie the Pooh',
             'country': 'pa',
             'summary': 'a fictional teddy bear created by A. A. Milne',
             'alias': ['Puh der Bär', 'Pooh Bear']
         }
     }, self.col)
     self.other = Entity.create({
         'schema': 'LegalEntity',
         'properties': {
             'name': 'Pu der Bär',
             'country': 'de',
             'description': 'he is a bear',
             'alias': ['Puh der Bär']
         }
     }, self.col)
     db.session.commit()

    def _generate(self):
        latest = Entity.latest()
        if self.latest is not None and self.latest >= latest:
            return

        self.latest = latest
        self.matches = defaultdict(set)

        q = Entity.all()
        q = q.options(joinedload('other_names'))
        q = q.filter(Entity.state == Entity.STATE_ACTIVE)
        for entity in q:
            for term in entity.regex_terms:
                self.matches[normalize_strong(term)].add(entity.id)

        self.regexes = []
        terms = self.matches.keys()
        terms = [t for t in terms if len(t) > 2]
        for i in count(0):
            terms_slice = terms[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]
            if not len(terms_slice):
                break
            body = '|'.join(terms_slice)
            rex = re.compile('( |^)(%s)( |$)' % body)
            # rex = re.compile('(%s)' % body)
            self.regexes.append(rex)

        log.info('Generating entity tagger: %r (%s terms)',
                 latest, len(terms))

def update(id):
    entity = obj_or_404(Entity.by_id(id))
    entity = Entity.save(get_data(entity=entity),
                         collection_id=entity.collection_id,
                         merge=arg_bool('merge'))
    db.session.commit()
    analyze_entity.delay(entity.id)
    return view(entity.id)

def cleanup_deleted():
    from aleph.model import Alert, Entity, Collection
    from aleph.model import Permission, Role
    Alert.cleanup_deleted()
    Permission.cleanup_deleted()
    Entity.cleanup_deleted()
    Collection.cleanup_deleted()
    Role.cleanup_deleted()
    db.session.commit()

def merge(id, other_id):
    entity = obj_or_404(Entity.by_id(id))
    check_authz(entity, authz.WRITE)
    other = obj_or_404(Entity.by_id(other_id))
    check_authz(other, authz.WRITE)
    entity.merge(other)
    db.session.commit()
    update_entity(entity)
    update_entity(other)
    return view(entity.id)

def update(id):
    entity = obj_or_404(Entity.by_id(id))
    check_authz(entity, authz.WRITE)
    data = request_data()
    data['id'] = entity.id
    possible_collections = authz.collections(authz.WRITE)
    possible_collections.extend([c.id for c in entity.collections])
    data['collections'] = [c for c in get_collections(data)
                           if c.id in possible_collections]
    entity = Entity.save(data, merge=arg_bool('merge'))
    db.session.commit()
    update_entity(entity)
    return view(entity.id)

 def setUp(self):
     super(EntitiesTestCase, self).setUp()
     self.rolex = self.create_user(foreign_id='user_3')
     self.col = Collection()
     self.col.label = 'Original Collection'
     self.col.foreign_id = 'test_coll_entities'
     db.session.add(self.col)
     self.col_other = Collection()
     self.col_other.label = 'Other Collection'
     self.col_other.foreign_id = 'test_coll_entities_other'
     db.session.add(self.col_other)
     db.session.flush()
     self.ent = Entity.save({
         'name': 'Winnie the Pooh',
         'collections': [self.col],
         'jurisdiction_code': 'pa',
         'summary': 'a fictional teddy bear created by author A. A. Milne',
         'identifiers': [{
             'scheme': 'wikipedia',
             'identifier': 'en:Winnie-the-Pooh'
         }],
         'other_names': [{
             'name': u'Puh der Bär'
         }, {
             'name': 'Pooh Bear'
         }]
     })
     db.session.add(self.ent)
     db.session.flush()
     self.other = Entity.save({
         'name': 'Pu der Bär',
         'collections': [self.col_other],
         'jurisdiction_code': 'de',
         'description': 'he is a bear',
         'identifiers': [{
             'scheme': 'wikipedia',
             'identifier': 'en:Winnie-the-Pooh'
         }, {
             'scheme': 'animals',
             'identifier': 'bears.winnie.pooh'
         }],
         'other_names': [{
             'name': u'Puh der Bär'
         }]
     })
     db.session.add(self.other)
     self.alert = Alert()
     self.alert.entity = self.other
     db.session.add(self.alert)
     db.session.commit()

def delete(id):
    entity = obj_or_404(Entity.by_id(id))
    authz.require(authz.watchlist_write(entity.watchlist_id))
    entity.delete()
    db.session.commit()
    analyze_entity.delay(id)
    return jsonify({"status": "ok"})

def load_entities():
    tx = get_graph().begin()
    q = Entity.all()
    q = q.filter(Entity.state == Entity.STATE_ACTIVE)
    for entity in q:
        load_entity(tx, entity)
    tx.commit()

def delete(id):
    entity = obj_or_404(Entity.by_id(id))
    check_authz(entity, authz.WRITE)
    entity.delete()
    db.session.commit()
    update_entity(entity)
    return jsonify({'status': 'ok'})

    def crawl_collection(self, collection):
        if not len(collection.get('subjects', [])):
            return
        url = urljoin(self.URL, '/api/collections/%s' % collection.get('id'))
        watchlist = Watchlist.by_foreign_id(url, {
            'label': collection.get('title')
        })
        res = requests.get('%s/permissions' % url, headers=self.HEADERS)
        for perm in res.json().get('results', []):
            Permission.grant_foreign(watchlist, perm.get('role'),
                                     perm.get('read'), perm.get('write'))

        log.info(" > Spindle collection: %s", watchlist.label)
        res = requests.get('%s/entities' % url, headers=self.HEADERS)
        previous_terms = watchlist.terms
        updated_terms = set()
        existing_entities = []
        for entity in res.json().get('results', []):
            if entity.get('name') is None:
                continue
            aliases = [on.get('alias') for on in entity.get('other_names', [])]
            ent = Entity.by_foreign_id(entity.get('id'), watchlist, {
                'name': entity.get('name'),
                'category': SCHEMATA.get(entity.get('$schema'), OTHER),
                'data': entity,
                'selectors': aliases
            })
            updated_terms.update(ent.terms)
            existing_entities.append(ent.id)
            log.info("  # %s (%s)", ent.name, ent.category)
        watchlist.delete_entities(spare=existing_entities)
        terms = previous_terms.symmetric_difference(updated_terms)
        self.emit_watchlist(watchlist, terms)

def format_results(query):
    sources = {}
    entities = {}
    results = []
    for row in raw_iter(query):
        src = row.get('_source')
        data = {}
        for name, value in src.items():
            if isinstance(value, dict) or name in SKIP_FIELDS:
                continue
            if name == 'entities':
                load_ids = []
                for entity_id in value:
                    if entity_id not in entities:
                        load_ids.append(entity_id)
                if len(load_ids):
                    for id, ent in Entity.by_id_set(load_ids).items():
                        entities[id] = ent.name

                value = ', '.join([entities.get(e) for e in value
                                   if entities.get(e) is not None])
            if isinstance(value, (list, tuple, set)):
                value = ', '.join(value)
            if name == 'source_id':
                # WARNING: don't to one query per row
                if value not in sources:
                    source = Source.by_id(value)
                    if source is None:
                        sources[value] = '[Deleted source %s]' % value
                    else:
                        sources[value] = source.label
                value = sources[value]
            data[name] = value
            results.append(data)
    return results

    def crawl(self):
        url = urljoin(self.host, '/ticket/all_closed/?format=json')
        collection = Collection.by_foreign_id(url, {
            'label': 'Investigative Dashboard Requests'
        })
        Permission.grant_foreign(collection, 'idashboard:occrp_staff',
                                 True, False)
        existing_entities = []
        terms = set()
        db.session.flush()
        for endpoint in ['all_closed', 'all_open']:
            url = urljoin(self.host, '/ticket/%s/?format=json' % endpoint)
            data = self.session.get(url).json()
            for req in data.get('paginator', {}).get('object_list'):
                category = REQUEST_TYPES.get(req.get('ticket_type'))
                if category is None:
                    continue
                ent = Entity.by_foreign_id(str(req.get('id')), collection, {
                    'name': req.get('name'),
                    'category': category,
                    'data': req,
                    'selectors': [req.get('name')]
                })
                terms.update(ent.terms)
                existing_entities.append(ent.id)
                log.info("  # %s (%s)", ent.name, ent.category)

        for entity in collection.entities:
            if entity.id not in existing_entities:
                entity.delete()
        self.emit_collection(collection, terms)

def test():
    from aleph.model import Entity
    graph = get_graph()
    tx = graph.begin()
    for entity_id in Entity.all_ids():
        remove_entity(tx, entity_id)
    tx.commit()

def load_document(tx, document):
    if tx is None:
        return
    log.info("Graph load [%s]: %r", document.id, document.meta)
    meta = document.meta
    node = DocumentNode.merge(tx, name=meta.title, alephTitle=document.type,
                              fileName=meta.file_name, fingerprint=document.id,
                              alephDocument=document.id)
    add_to_collections(tx, node, document.collections,
                       alephDocument=document.id)

    for email in meta.emails:
        enode = EmailNode.merge(tx, name=email, fingerprint=email)
        MENTIONS.merge(tx, node, enode, alephDocument=document.id)
        add_to_collections(tx, enode, document.collections,
                           alephDocument=document.id)

    for phone in meta.phone_numbers:
        pnode = PhoneNode.merge(tx, name=phone, fingerprint=phone)
        MENTIONS.merge(tx, node, pnode, alephDocument=document.id)
        add_to_collections(tx, pnode, document.collections,
                           alephDocument=document.id)

    for entity in Entity.all_by_document(document.id):
        enode = load_entity(tx, entity)
        MENTIONS.merge(tx, node, enode,
                       alephDocument=document.id,
                       alephEntity=entity.id)
    return node