Java BatchInserterIndex.add方法代码示例

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
@Ignore
@Test
public void testInsertionSpeed() throws Exception
{
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex index = provider.nodeIndex( "yeah", EXACT_CONFIG );
    index.setCacheCapacity( "key", 1000000 );
    long t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        Map<String, Object> properties = map( "key", "value" + i );
        long id = inserter.createNode( properties );
        index.add( id, properties );
    }
    System.out.println( "insert:" + ( currentTimeMillis() - t ) );
    index.flush();

    t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        count( (Iterator<Long>) index.get( "key", "value" + i ) );
    }
    System.out.println( "get:" + ( currentTimeMillis() - t ) );
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
@Test
public void testBatchIndexToAutoIndex() throws IOException {
  BatchInserter inserter = BatchInserters.inserter(new File(path));
  BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
  BatchInserterIndex index =
      indexProvider.nodeIndex("node_auto_index", MapUtil.stringMap("type", "exact"));
  long node = inserter.createNode(MapUtil.map("foo", "bar"));
  index.add(node, MapUtil.map("foo", "bar"));
  index.flush();
  assertThat("Batch indexed node can be retrieved", index.get("foo", "bar").next(), is(node));
  indexProvider.shutdown();
  inserter.shutdown();
  graphDb = getGraphDb();
  try (Transaction tx = graphDb.beginTx()) {
    assertThat("AutoIndex is not enabled after reopening the graph", graphDb.index()
        .getNodeAutoIndexer().isEnabled(), is(false));
    assertThat("AutoIndexed properties are not maintained after closing the graph", graphDb
        .index().getNodeAutoIndexer().getAutoIndexedProperties(), is(empty()));
    assertThat("Batch index properties are in the index", graphDb.index().getNodeAutoIndexer()
        .getAutoIndex().query("foo", "bar").size(), is(1));
    tx.success();
  }
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
private static long getOrCreateUserNodeID(final String userid,
                                          final Map<String, Long> usersNodesMap,
                                          final BatchInserterIndex usersIndex,
                                          final BatchInserter inserter,
                                          final Label label) {

    long userNodeID;

    if (usersNodesMap.containsKey(userid)) {
        userNodeID = usersNodesMap.get(userid);
    } else {
        userNodeID = inserter.createNode(MapUtil.map("node_id", userid), label);
        usersNodesMap.put(userid, userNodeID);
        usersIndex.add(userNodeID, MapUtil.map("node_id", userid));
    }

    return userNodeID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
private static long getOrCreateItemNodeID(final String itemid,
                                          final Map<String, Long> itemsNodesMap,
                                          final BatchInserterIndex itemsIndex,
                                          final BatchInserter inserter,
                                          final Label label) {

    long itemNodeID;

    if (itemsNodesMap.containsKey(itemid)) {
        itemNodeID = itemsNodesMap.get(itemid);
    } else {
        itemNodeID = inserter.createNode(MapUtil.map("node_id", itemid), label);
        itemsNodesMap.put(itemid, itemNodeID);
        itemsIndex.add(itemNodeID, MapUtil.map("node_id", itemid));
    }

    return itemNodeID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
private static long getOrCreateRelatinshipID(final String timestamp,
                                             final long userNodeid,
                                             final long itemNodeid,
                                             final Map<String, Long> relationshipsMap,
                                             final BatchInserterIndex relationshipIndex,
                                             final BatchInserter inserter,
                                             final RelationshipType relType) {

    long relationID;
    String uniqueRelationID = userNodeid + itemNodeid + timestamp;
    if (relationshipsMap.containsKey(uniqueRelationID)) {
        relationID = relationshipsMap.get(uniqueRelationID);
    } else {

        relationID = inserter.createRelationship(userNodeid, itemNodeid, relType,
                MapUtil.map("timestamp", timestamp));

        relationshipsMap.put(uniqueRelationID, relationID);
        relationshipIndex.add(relationID, MapUtil.map("useritem", uniqueRelationID));
    }

    return relationID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
private synchronized void createNode(String labelName, Map<String, Object> properties, BatchInserterIndex idx,
		String pk) {
	idx.add(this.inserter.createNode(properties, DynamicLabel.label(labelName)),
			MapUtil.map(pk, properties.get(pk)));
	long n = this.counter.get(labelName);
	this.counter.put(labelName, ++n);
	if(n % this.flushInterval == 0)
		idx.flush();
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
/**
 * Push a node importId into the batch index.
 *
 * @param indexName     The batch index name
 * @param node          Neo4j identifier
 * @param importIdValue Import identifier
 */
public void indexNodeInBatchIndex(String indexName, long node, Object importIdValue) {
    if (batchInserterIndexes.containsKey(indexName)) {
        BatchInserterIndex index = batchInserterIndexes.get(indexName);

        Map<String, Object> props = new HashMap<>();
        props.put(BACTH_INDEX_ID_NAME, importIdValue);
        index.add(node, props);
    } else {
        log.trace("Can't index node " + node + "/" + importIdValue + " into a none existant index [" + indexName + "]");
    }
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
@Test
public void batchInsert() throws Exception
{
    Neo4jTestCase.deleteFileOrDirectory( new File(
            "target/neo4jdb-batchinsert" ) );
    // START SNIPPET: batchInsert
    BatchInserter inserter = BatchInserters.inserter( "target/neo4jdb-batchinsert" );
    BatchInserterIndexProvider indexProvider =
            new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex actors =
            indexProvider.nodeIndex( "actors", MapUtil.stringMap( "type", "exact" ) );
    actors.setCacheCapacity( "name", 100000 );

    Map<String, Object> properties = MapUtil.map( "name", "Keanu Reeves" );
    long node = inserter.createNode( properties );
    actors.add( node, properties );

    //make the changes visible for reading, use this sparsely, requires IO!
    actors.flush();

    // Make sure to shut down the index provider as well
    indexProvider.shutdown();
    inserter.shutdown();
    // END SNIPPET: batchInsert

    GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( "target/neo4jdb-batchinsert" );
    try ( Transaction tx = db.beginTx() )
    {
        Index<Node> index = db.index().forNodes( "actors" );
        Node reeves = index.get( "name", "Keanu Reeves" ).next();
        assertEquals( node, reeves.getId() );
    }
    db.shutdown();
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
/**
 * @param indexName 
 * @param node
 */
public static void addToLegacyNodeIndex(Neo4ArtLegacyIndex neo4ArtLegacyIndex, Node node)
{
  BatchInserterIndex batchInserterIndex = getBatchInserterIndexProviderInstance().nodeIndex(neo4ArtLegacyIndex.getName(), getLegacyNodeIndexConfig(neo4ArtLegacyIndex.getType()));
  
  if (batchInserterIndex == null)
  {
    throw new IllegalArgumentException("Index " + neo4ArtLegacyIndex.getName() + " doesn't exists. You must create it before adding something to it.");
  }    
  
  batchInserterIndex.add(node.getNodeId(), node.getProperties());
}
 

import org.neo4j.unsafe.batchinsert.BatchInserterIndex; //导入方法依赖的package包/类
private void copyNFlushNClearIndex(final ObjectLongOpenHashMap<String> tempIndex, final BatchInserterIndex neo4jIndex,
                                   final String indexProperty, final String indexName) {

	BatchNeo4jProcessor.LOG.debug("start pumping '{}' index of size '{}'", indexName, tempIndex.size());

	final Object[] keys = tempIndex.keys;
	final long[] values = tempIndex.values;
	final boolean[] states = tempIndex.allocated;

	BatchNeo4jProcessor.LOG.debug("keys size = '{}' :: values size = '{}' :: states size = '{}'", keys.length, values.length, states.length);

	int j = 0;
	long tick = System.currentTimeMillis();
	int sinceLast = 0;

	for (int i = 0; i < states.length; i++) {

		if (states[i]) {

			neo4jIndex.add(values[i], MapUtil.map(indexProperty, keys[i].toString()));

			j++;

			final int entryDelta = j - sinceLast;
			final long timeDelta = (System.currentTimeMillis() - tick) / 1000;

			if (entryDelta >= 1000000 || timeDelta >= 60) {

				sinceLast = j;
				final double duration = (double) entryDelta / timeDelta;

				BatchNeo4jProcessor.LOG.debug("wrote '{}' entries @ ~{} entries/second.", j, duration);

				tick = System.currentTimeMillis();
			}
		}
	}

	BatchNeo4jProcessor.LOG.debug("finished pumping '{}' index; wrote '{}' entries", indexName, j);

	BatchNeo4jProcessor.LOG.debug("start flushing and clearing index");

	neo4jIndex.flush();
	tempIndex.clear();

	BatchNeo4jProcessor.LOG.debug("finished flushing and clearing index");
}