Python compat.u函数代码示例

        def check_update(update_spec):
            updated = self.conn.test.command(
                SON([('findAndModify', 'test'),
                     ('query', {"a": 0}),
                     ('update', update_spec),
                     ('new', True)]))['value']

            # Stringify _id to match what will be retrieved from Solr
            updated[u('_id')] = u(updated['_id'])
            # Flatten the MongoDB document to match Solr
            updated = docman._clean_doc(updated, 'dummy.namespace', 0)
            # Allow some time for update to propagate
            time.sleep(3)
            replicated = list(self._search("a:0"))[0]

            # Remove add'l fields until these are stored in a separate Solr core
            updated.pop('_ts')
            replicated.pop('_ts')
            updated.pop('ns')
            replicated.pop('ns')

            # Remove field added by Solr
            replicated.pop("_version_")

            self.assertEqual(replicated, updated)

    def remove(self, document_id, namespace, timestamp):
        """Remove a document from Elasticsearch."""
        index, doc_type = self._index_and_mapping(namespace)

        if doc_type in self.routing and 'parentField' in self.routing[doc_type]:
            # We can't use delete() directly here and have to do a full search first.
            # This is due to the fact that Elasticsearch needs the parent ID to
            # know where to route the delete request. We might not have the parent
            # ID available in our remove request though.
            document = self._search_doc_by_id(index, doc_type, document_id)
            if document is None:
                LOG.error('Could not find document with ID "%s" in Elasticsearch to apply remove', u(document_id))
                return
            parent_id = self._get_parent_id(doc_type, document)
            self.elastic.delete(index=index, doc_type=doc_type,
                                id=u(document_id), parent=parent_id,
                                refresh=(self.auto_commit_interval == 0))
        else:
            self.elastic.delete(index=index, doc_type=doc_type,
                                id=u(document_id),
                                refresh=(self.auto_commit_interval == 0))

        self.elastic.delete(index=self.meta_index_name, doc_type=self.meta_type,
                            id=u(document_id),
                            refresh=(self.auto_commit_interval == 0))

 def remove(self, doc):
     """Remove a document from Elasticsearch."""
     self.elastic.delete(index=doc['ns'], doc_type=self.doc_type,
                         id=u(doc["_id"]),
                         refresh=(self.auto_commit_interval == 0))
     self.elastic.delete(index=self.meta_index_name, doc_type=self.meta_type,
                         id=u(doc["_id"]),
                         refresh=(self.auto_commit_interval == 0))

 def remove(self, document_id, namespace, timestamp):
     """Remove a document from Elasticsearch."""
     index, doc_type = self._index_and_mapping(namespace)
     self.elastic.delete(index=index, doc_type=doc_type,
                         id=u(document_id),
                         refresh=(self.auto_commit_interval == 0))
     self.elastic.delete(index=self.meta_index_name, doc_type=self.meta_type,
                         id=u(document_id),
                         refresh=(self.auto_commit_interval == 0))

    def upsert(self, doc, namespace, timestamp):
        """Inserts a document into Neo4j."""
        index, doc_type = self._index_and_mapping(namespace)
        doc_id = u(doc.pop("uid"))
        metadata = { "_ts": timestamp }
        doc = self._formatter.format_document(doc)
        builder = NodesAndRelationshipsBuilder(doc, doc_type, doc_id, metadata)
        self.apply_id_constraint(builder.doc_types)
        tx = self.graph.cypher.begin()
        for statement in builder.query_nodes.keys():
            tx.append(statement, builder.query_nodes[statement])
        for query in builder.cypher_list:
            tx.append(query)
            # Adding cyphers from cypher list
        for relationship, params in builder.relationships_query:
            tx.append(relationship, params)
        for statement in builder.statements_with_params:
            for key in statement.keys():
                tx.append(key, statement[key])
        commit_result = None
        try:
            commit_result = tx.commit()
            print commit_result
        except Exception as e:
            LOG.error('{}'.format(e.message))
            pass

        if commit_result:
            nodeids_list = self._get_nodeids(commit_result)
            self.create_geospatial_indices(nodeids_list)

 def bulk_upsert(self, docs, namespace, timestamp):
     """Insert multiple documents into Neo4j."""
     """Maximum chunk size is 1000. Transaction blocks won't have more than 1000 statements."""
     metadata = { "_ts": timestamp }
     tx = self.graph.cypher.begin()
     for doc in docs:
         index, doc_type = self._index_and_mapping(namespace)
         doc_id = u(doc.pop("uid"))
         doc = self._formatter.format_document(doc)
         builder = NodesAndRelationshipsBuilder(doc, doc_type, doc_id, metadata)
         self.apply_id_constraint(builder.doc_types)
         for statement in builder.query_nodes.keys():
             tx.append(statement, builder.query_nodes[statement])
         for query in builder.cypher_list:
             tx.append(query)
             # Adding cyphers from cypher list
         for relationship, params in builder.relationships_query:
             tx.append(relationship, params)
         for statement in builder.statements_with_params:
             for key in statement.keys():
                 tx.append(key, statement[key])
     try:
         tx.commit()
     except Exception as e:
         LOG.error('{}'.format(e.message))
         pass

 def _upsert(self, cursor, doc, namespace, timestamp):
     doc_id = compat.u(doc["_id"])
     log.debug("Upsert %s into %s", doc_id, namespace)
     cursor.execute(u"""INSERT INTO "{table}" ("{id}", _ts, document) VALUES (%(id)s, %(ts)s, %(doc)s) """
                    u"""ON CONFLICT ("{id}") """
                    u"""DO UPDATE SET (_ts, document) = (%(ts)s, %(doc)s);""".format(table=namespace, id=self.unique_key),
                    {"id": doc_id, "ts": timestamp, "doc": psycopg2.extras.Json(self._formatter.format_document(doc))})

    def update(self, document_id, update_spec, namespace, timestamp):
        """Apply updates given in update_spec to the document whose id
        matches that of doc.
        """

        index, doc_type = self._index_and_mapping(namespace)
        with self.lock:
            # Check if document source is stored in local buffer
            document = self.BulkBuffer.get_from_sources(index,
                                                        doc_type,
                                                        u(document_id))
        if document:
            # Document source collected from local buffer
            # Perform apply_update on it and then it will be
            # ready for commiting to Elasticsearch
            updated = self.apply_update(document, update_spec)
            # _id is immutable in MongoDB, so won't have changed in update
            updated['_id'] = document_id
            self.upsert(updated, namespace, timestamp)
        else:
            # Document source needs to be retrieved from Elasticsearch
            # before performing update. Pass update_spec to upsert function
            updated = {"_id": document_id}
            self.upsert(updated, namespace, timestamp, update_spec)
        # upsert() strips metadata, so only _id + fields in _source still here
        return updated

    def update(self, document_id, update_spec, namespace, timestamp):
        """Apply updates given in update_spec to the document whose id
        matches that of doc.
        """
        self.commit()
        index, doc_type = self._index_and_mapping(namespace)

        if doc_type in self.routing and 'parentField' in self.routing[doc_type]:
            # We can't use get() here and have to do a full search instead.
            # This is due to the fact that Elasticsearch needs the parent ID to
            # know where to route the get request. We might not have the parent
            # ID available in our update request though.
            document = self._search_doc_by_id(index, doc_type, document_id)
            if document is None:
                LOG.error('Could not find document with ID "%s" in Elasticsearch to apply update', u(document_id))
                return None
        else:
            document = self.elastic.get(index=index, doc_type=doc_type,
                                        id=u(document_id))

        updated = self.apply_update(document['_source'], update_spec)
        # _id is immutable in MongoDB, so won't have changed in update
        updated['_id'] = document['_id']
        if '_parent' in document:
            updated['_parent'] = document['_parent']
        self.upsert(updated, namespace, timestamp)
        # upsert() strips metadata, so only _id + fields in _source still here
        return updated

        def iterate_chunks():
            more_chunks = True

            while more_chunks:
                tx = self.graph.cypher.begin()
                metadata = {"_ts": timestamp}
                for i in range(self.chunk_size):
                    try:
                        doc = next(docs)
                        index, doc_type = self._index_and_mapping(namespace)
                        doc_id = u(doc.pop("_id"))
                        doc = self._formatter.format_document(doc)
                        builder = NodesAndRelationshipsBuilder(doc, doc_type, doc_id, metadata)
                        self.apply_id_constraint(builder.doc_types)
                        for statement in builder.query_nodes.keys():
                            tx.append(statement, builder.query_nodes[statement])
                        for relationship in builder.relationships_query.keys():
                            tx.append(relationship, builder.relationships_query[relationship])
                    except StopIteration:
                        more_chunks = False
                        if i > 0:
                            yield tx
                        break
                if more_chunks:
                    yield tx

 def build_nodes_query(self, doc_type, document, id):
   self.doc_types.append(doc_type)
   parameters = {'_id':id}
   if self.is_dict(self.metadata):
     parameters.update(self.metadata) 
   for key in document.keys():
     if self.is_reference(key):
       self.build_node_with_reference(doc_type, key, id, document[key])
       continue
     if self.is_objectid(document[key]):
       parameters.update({key: u(document[key])})
       self.build_node_with_reference(doc_type, key, id, document[key])
       continue
     #TODO: handle arrays of ObjectIds
     if document[key] is None:
       continue
     elif self.is_dict(document[key]):
       self.build_relationships_query(doc_type, key, id, id)
       self.build_nodes_query(key, document[key], id)
     elif self.is_json_array(document[key]):
       for json in self.format_params(document[key]):
         json_key = key + str(document[key].index(json))
         self.build_relationships_query(doc_type, json_key, id, id)
         self.build_nodes_query(json_key, json, id)
     elif self.is_multimensional_array(document[key]):
       parameters.update(self.flatenned_property(key, document[key]))
     else:
       parameters.update({ key: self.format_params(document[key]) })
   query = "CREATE (c:Document:`{doc_type}` {{parameters}})".format(doc_type=doc_type)
   self.query_nodes.update({query: {"parameters":parameters}})

    def remove(self, document_id, namespace, timestamp):
        """Removes documents from Solr

        The input is a python dictionary that represents a mongo document.
        """
        self.solr.delete(id=u(document_id),
                         commit=(self.auto_commit_interval == 0))

 def docs_to_upsert():
     doc = None
     for doc in docs:
         # Remove metadata and redundant _id
         index, doc_type = self._index_and_mapping(namespace)
         doc_id = u(doc.pop("_id"))
         document_action = {
             "_index": index,
             "_type": doc_type,
             "_id": doc_id,
             "_source": self._formatter.format_document(doc)
         }
         document_meta = {
             "_index": self.meta_index_name,
             "_type": self.meta_type,
             "_id": doc_id,
             "_source": {
                 "ns": namespace,
                 "_ts": timestamp
             }
         }
         yield document_action
         yield document_meta
     if doc is None:
         raise errors.EmptyDocsError(
             "Cannot upsert an empty sequence of "
             "documents into Elastic Search")

    def upsert(self, doc, namespace, timestamp):
        """Insert a document into Elasticsearch."""
        index, doc_type = self._index_and_mapping(namespace)
        # No need to duplicate '_id' in source document
        doc_id = u(doc.pop("_id"))
        metadata = {
            "ns": namespace,
            "_ts": timestamp
        }

        parent_id = self._get_parent_id(doc_type, doc)
        # Index the source document, using lowercase namespace as index name.
        if parent_id is None:
            self.elastic.index(index=index, doc_type=doc_type,
                               body=self._formatter.format_document(doc), id=doc_id,
                               refresh=(self.auto_commit_interval == 0))
        else:
            self.elastic.index(index=index, doc_type=doc_type,
                               body=self._formatter.format_document(doc), id=doc_id,
                               parent=parent_id, refresh=(self.auto_commit_interval == 0))

        # Index document metadata with original namespace (mixed upper/lower).
        self.elastic.index(index=self.meta_index_name, doc_type=self.meta_type,
                           body=bson.json_util.dumps(metadata), id=doc_id,
                           refresh=(self.auto_commit_interval == 0))
        # Leave _id, since it's part of the original document
        doc['_id'] = doc_id

    def update(self, document_id, update_spec, namespace, timestamp):
        """Apply updates given in update_spec to the document whose id
        matches that of doc.

        """
        # Commit outstanding changes so that the document to be updated is the
        # same version to which the changes apply.
        self.commit()
        # Need to escape special characters in the document_id.
        document_id = ''.join(map(
            lambda c: '\\' + c if c in ESCAPE_CHARACTERS else c,
            u(document_id)
        ))

        query = "%s:%s" % (self.unique_key, document_id)
        results = self.solr.search(query)
        if not len(results):
            # Document may not be retrievable yet
            self.commit()
            results = self.solr.search(query)
        # Results is an iterable containing only 1 result
        for doc in results:
            # Remove metadata previously stored by Mongo Connector.
            doc.pop('ns')
            doc.pop('_ts')
            updated = self.apply_update(doc, update_spec)
            # A _version_ of 0 will always apply the update
            updated['_version_'] = 0
            self.upsert(updated, namespace, timestamp)
            return updated