Java BatchInserter类代码示例

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void shouldNotIndexNodesWithWrongLabel() throws Exception
{
    // Given
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );

    inserter.createNode( map("name", "Bob"), label( "User" ), label("Admin"));

    inserter.createDeferredSchemaIndex( label( "Banana" ) ).on( "name" ).create();

    // When
    inserter.shutdown();

    // Then
    GraphDatabaseService db = dbRule.getGraphDatabaseService();
    try(Transaction tx = db.beginTx())
    {
        assertThat( count( db.findNodes( label( "Banana" ), "name", "Bob" ) ), equalTo(0) );
    }
    finally
    {
        db.shutdown();
    }

}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void shouldBeAbleToMakeRepeatedCallsToSetNodeProperty() throws Exception
{
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );
    long nodeId = inserter.createNode( Collections.<String, Object>emptyMap() );

    final Object finalValue = 87;
    inserter.setNodeProperty( nodeId, "a", "some property value" );
    inserter.setNodeProperty( nodeId, "a", 42 );
    inserter.setNodeProperty( nodeId, "a", 3.14 );
    inserter.setNodeProperty( nodeId, "a", true );
    inserter.setNodeProperty( nodeId, "a", finalValue );
    inserter.shutdown();

    GraphDatabaseService db = dbRule.getGraphDatabaseService();
    try(Transaction ignored = db.beginTx())
    {
        assertThat( db.getNodeById( nodeId ).getProperty( "a" ), equalTo( finalValue ) );
    }
    finally
    {
        db.shutdown();
    }
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Ignore
@Test
public void testInsertionSpeed() throws Exception
{
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex index = provider.nodeIndex( "yeah", EXACT_CONFIG );
    index.setCacheCapacity( "key", 1000000 );
    long t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        Map<String, Object> properties = map( "key", "value" + i );
        long id = inserter.createNode( properties );
        index.add( id, properties );
    }
    System.out.println( "insert:" + ( currentTimeMillis() - t ) );
    index.flush();

    t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        count( (Iterator<Long>) index.get( "key", "value" + i ) );
    }
    System.out.println( "get:" + ( currentTimeMillis() - t ) );
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void testBatchIndexToAutoIndex() throws IOException {
  BatchInserter inserter = BatchInserters.inserter(new File(path));
  BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
  BatchInserterIndex index =
      indexProvider.nodeIndex("node_auto_index", MapUtil.stringMap("type", "exact"));
  long node = inserter.createNode(MapUtil.map("foo", "bar"));
  index.add(node, MapUtil.map("foo", "bar"));
  index.flush();
  assertThat("Batch indexed node can be retrieved", index.get("foo", "bar").next(), is(node));
  indexProvider.shutdown();
  inserter.shutdown();
  graphDb = getGraphDb();
  try (Transaction tx = graphDb.beginTx()) {
    assertThat("AutoIndex is not enabled after reopening the graph", graphDb.index()
        .getNodeAutoIndexer().isEnabled(), is(false));
    assertThat("AutoIndexed properties are not maintained after closing the graph", graphDb
        .index().getNodeAutoIndexer().getAutoIndexedProperties(), is(empty()));
    assertThat("Batch index properties are in the index", graphDb.index().getNodeAutoIndexer()
        .getAutoIndex().query("foo", "bar").size(), is(1));
    tx.success();
  }
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
/**
 * 
 * @param graph2
 */
private static void registerShutdownHook(final BatchInserter graph2) {
	if (lastBatchGraph == null) {
		Runtime.getRuntime().addShutdownHook(new Thread() {
			@Override
			public void run() {
				try {
					final long l = System.nanoTime();
					lastBatchGraph.shutdown();
					LOGGER.info("SHUTDOWN HOOK INVOKED: (took ~{}sec to commit changes", (System.nanoTime() - l) / 1_000_000_000);
				} catch (Exception e) {
					LOGGER.error("Error during shutdown hook", e);
				}
			}
		});
	}
	lastBatchGraph = graph2;
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
public BatchNeo4jProcessor(final BatchInserter inserter) throws DMPGraphException {

		this.inserter = inserter;

		BatchNeo4jProcessor.LOG.debug("start writing");

		bnodes = new ObjectLongOpenHashMap<>();
		nodeResourceMap = new LongLongOpenHashMap();

		tempResourcesIndex = new ObjectLongOpenHashMap<>();
		tempResourcesWDataModelIndex = new ObjectLongOpenHashMap<>();
		tempResourceTypes = new ObjectLongOpenHashMap<>();
		//tempStatementHashes = new LongLongOpenHashMap();
		final Tuple<Set<Long>, DB> mapDBTuple = MapDBUtils
				.createOrGetInMemoryLongIndexTreeSetNonTransactional(GraphIndexStatics.TEMP_STATEMENT_HASHES_INDEX_NAME);
		tempStatementHashes = mapDBTuple.v1();
		tempStatementHashesDB = mapDBTuple.v2();

		// TODO: init all indices, when batch inserter should work on a pre-filled database (otherwise, the existing index would
		// utilised in the first run)
		// initIndices();
		initValueIndex();
		initStatementIndex();
	}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
private static long getOrCreateUserNodeID(final String userid,
                                          final Map<String, Long> usersNodesMap,
                                          final BatchInserterIndex usersIndex,
                                          final BatchInserter inserter,
                                          final Label label) {

    long userNodeID;

    if (usersNodesMap.containsKey(userid)) {
        userNodeID = usersNodesMap.get(userid);
    } else {
        userNodeID = inserter.createNode(MapUtil.map("node_id", userid), label);
        usersNodesMap.put(userid, userNodeID);
        usersIndex.add(userNodeID, MapUtil.map("node_id", userid));
    }

    return userNodeID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
private static long getOrCreateItemNodeID(final String itemid,
                                          final Map<String, Long> itemsNodesMap,
                                          final BatchInserterIndex itemsIndex,
                                          final BatchInserter inserter,
                                          final Label label) {

    long itemNodeID;

    if (itemsNodesMap.containsKey(itemid)) {
        itemNodeID = itemsNodesMap.get(itemid);
    } else {
        itemNodeID = inserter.createNode(MapUtil.map("node_id", itemid), label);
        itemsNodesMap.put(itemid, itemNodeID);
        itemsIndex.add(itemNodeID, MapUtil.map("node_id", itemid));
    }

    return itemNodeID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
private static long getOrCreateRelatinshipID(final String timestamp,
                                             final long userNodeid,
                                             final long itemNodeid,
                                             final Map<String, Long> relationshipsMap,
                                             final BatchInserterIndex relationshipIndex,
                                             final BatchInserter inserter,
                                             final RelationshipType relType) {

    long relationID;
    String uniqueRelationID = userNodeid + itemNodeid + timestamp;
    if (relationshipsMap.containsKey(uniqueRelationID)) {
        relationID = relationshipsMap.get(uniqueRelationID);
    } else {

        relationID = inserter.createRelationship(userNodeid, itemNodeid, relType,
                MapUtil.map("timestamp", timestamp));

        relationshipsMap.put(uniqueRelationID, relationID);
        relationshipIndex.add(relationID, MapUtil.map("useritem", uniqueRelationID));
    }

    return relationID;

}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
public LuceneBatchInserterIndexProviderNewImpl( final BatchInserter inserter )
{
    this.inserter = inserter;
    this.indexStore = ((BatchInserterImpl) inserter).getIndexStore();
    this.relationshipLookup = new LuceneBatchInserterIndex.RelationshipLookup()
    {
        @Override
        public EntityId lookup( long id )
        {
            // TODO too may objects allocated here
            BatchRelationship rel = inserter.getRelationshipById( id );
            return new RelationshipData( id, rel.getStartNode(), rel.getEndNode() );
        }
    };
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void batchInsert() throws Exception
{
    Neo4jTestCase.deleteFileOrDirectory( new File(
            "target/neo4jdb-batchinsert" ) );
    // START SNIPPET: batchInsert
    BatchInserter inserter = BatchInserters.inserter( "target/neo4jdb-batchinsert" );
    BatchInserterIndexProvider indexProvider =
            new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex actors =
            indexProvider.nodeIndex( "actors", MapUtil.stringMap( "type", "exact" ) );
    actors.setCacheCapacity( "name", 100000 );

    Map<String, Object> properties = MapUtil.map( "name", "Keanu Reeves" );
    long node = inserter.createNode( properties );
    actors.add( node, properties );

    //make the changes visible for reading, use this sparsely, requires IO!
    actors.flush();

    // Make sure to shut down the index provider as well
    indexProvider.shutdown();
    inserter.shutdown();
    // END SNIPPET: batchInsert

    GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( "target/neo4jdb-batchinsert" );
    try ( Transaction tx = db.beginTx() )
    {
        Index<Node> index = db.index().forNodes( "actors" );
        Node reeves = index.get( "name", "Keanu Reeves" ).next();
        assertEquals( node, reeves.getId() );
    }
    db.shutdown();
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void shouldIndexNodesWithMultipleLabels() throws Exception
{
    // Given
    String path = dbRule.getStoreDirAbsolutePath();
    BatchInserter inserter = BatchInserters.inserter( path );

    inserter.createNode( map( "name", "Bob" ), label( "User" ), label( "Admin" ) );

    inserter.createDeferredSchemaIndex( label( "User" ) ).on( "name" ).create();
    inserter.createDeferredSchemaIndex( label( "Admin" ) ).on( "name" ).create();

    // When
    inserter.shutdown();

    // Then
    GraphDatabaseService db = dbRule.getGraphDatabaseService();
    try ( Transaction tx = db.beginTx() )
    {
        assertThat( count( db.findNodes( label( "User" ), "name", "Bob" ) ), equalTo(1) );
        assertThat( count( db.findNodes( label( "Admin" ), "name", "Bob" ) ), equalTo(1) );
    }
    finally
    {
        db.shutdown();
    }

}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
@Test
public void shouldBeAbleToMakeRepeatedCallsToSetNodePropertyWithMultiplePropertiesPerBlock() throws Exception
{
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );
    long nodeId = inserter.createNode( Collections.<String, Object>emptyMap() );

    final Object finalValue1 = 87;
    final Object finalValue2 = 3.14;
    inserter.setNodeProperty( nodeId, "a", "some property value" );
    inserter.setNodeProperty( nodeId, "a", 42 );
    inserter.setNodeProperty( nodeId, "b", finalValue2 );
    inserter.setNodeProperty( nodeId, "a", finalValue2 );
    inserter.setNodeProperty( nodeId, "a", true );
    inserter.setNodeProperty( nodeId, "a", finalValue1 );
    inserter.shutdown();

    GraphDatabaseService db = dbRule.getGraphDatabaseService();
    try(Transaction ignored = db.beginTx())
    {
        assertThat( db.getNodeById( nodeId ).getProperty( "a" ), equalTo( finalValue1 ) );
        assertThat( db.getNodeById( nodeId ).getProperty( "b" ), equalTo( finalValue2 ) );
    }
    finally
    {
        db.shutdown();
    }
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
/**
 * 
 * @return
 * @return
 */
private static BatchInserter getBatchInserterInstance()
{
  Map<String, String> config = new HashMap<>();
  config.put("dbms.pagecache.memory", "16G");

  return getBatchInserterInstance(config);
}
 

import org.neo4j.unsafe.batchinsert.BatchInserter; //导入依赖的package包/类
/**
 * 
 * @param node
 * @return
 */
public static long createNode(Node node)
{
  BatchInserter batchInserter = getBatchInserterInstance();

  long nodeId = batchInserter.createNode(node.getProperties(), node.getLabels());
  
  node.setNodeId(nodeId);
  
  return nodeId;
}