Java LuceneBatchInserterIndexProvider类代码示例

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
@Ignore
@Test
public void testInsertionSpeed() throws Exception
{
    BatchInserter inserter = BatchInserters.inserter( dbRule.getStoreDirAbsolutePath() );
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex index = provider.nodeIndex( "yeah", EXACT_CONFIG );
    index.setCacheCapacity( "key", 1000000 );
    long t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        Map<String, Object> properties = map( "key", "value" + i );
        long id = inserter.createNode( properties );
        index.add( id, properties );
    }
    System.out.println( "insert:" + ( currentTimeMillis() - t ) );
    index.flush();

    t = currentTimeMillis();
    for ( int i = 0; i < 1000000; i++ )
    {
        count( (Iterator<Long>) index.get( "key", "value" + i ) );
    }
    System.out.println( "get:" + ( currentTimeMillis() - t ) );
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
@Test
public void testBatchIndexToAutoIndex() throws IOException {
  BatchInserter inserter = BatchInserters.inserter(new File(path));
  BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
  BatchInserterIndex index =
      indexProvider.nodeIndex("node_auto_index", MapUtil.stringMap("type", "exact"));
  long node = inserter.createNode(MapUtil.map("foo", "bar"));
  index.add(node, MapUtil.map("foo", "bar"));
  index.flush();
  assertThat("Batch indexed node can be retrieved", index.get("foo", "bar").next(), is(node));
  indexProvider.shutdown();
  inserter.shutdown();
  graphDb = getGraphDb();
  try (Transaction tx = graphDb.beginTx()) {
    assertThat("AutoIndex is not enabled after reopening the graph", graphDb.index()
        .getNodeAutoIndexer().isEnabled(), is(false));
    assertThat("AutoIndexed properties are not maintained after closing the graph", graphDb
        .index().getNodeAutoIndexer().getAutoIndexedProperties(), is(empty()));
    assertThat("Batch index properties are in the index", graphDb.index().getNodeAutoIndexer()
        .getAutoIndex().query("foo", "bar").size(), is(1));
    tx.success();
  }
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
@Test
public void cachesShouldBeFilledWhenAddToMultipleIndexesCreatedNow() throws Exception
{
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex index = provider.nodeIndex( "index1", LuceneIndexImplementation.EXACT_CONFIG );
    index.setCacheCapacity( "name", 100000 );
    String nameKey = "name";
    String titleKey = "title";

    assertCacheIsEmpty( index, nameKey, titleKey );
    index.add( 0, map( "name", "Neo", "title", "Matrix" ) );
    assertCacheContainsSomething( index, nameKey );
    assertCacheIsEmpty( index, titleKey );

    BatchInserterIndex index2 = provider.nodeIndex( "index2", LuceneIndexImplementation.EXACT_CONFIG );
    index2.setCacheCapacity( "title", 100000 );
    assertCacheIsEmpty( index2, nameKey, titleKey );
    index2.add( 0, map( "name", "Neo", "title", "Matrix" ) );
    assertCacheContainsSomething( index2, titleKey );
    assertCacheIsEmpty( index2, nameKey );

    provider.shutdown();
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
public NeoDb(String dbPath, long flushInterval, int cacheCapacity) throws IOException {
	this.flushInterval = flushInterval;
	this.cacheCapacity = cacheCapacity;
	File tempStoreDir = new File(dbPath);
	FileUtils.deleteRecursively(tempStoreDir);
	this.inserter = BatchInserters.inserter(tempStoreDir);
	this.indexProvider = new LuceneBatchInserterIndexProvider(this.inserter);
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
/**
 * Constructor that create an embedded database.
 *
 * @param path          Path of the graph database folder
 * @param configuration Configuration map of database
 */
public Neo4jBatchDatabase(String path, Map<String, String> configuration) {
    try {
        // Initialize batch inserter
        this.inserter = BatchInserters.inserter(new File(path), configuration);

        // Init
        this.indexProvider = new LuceneBatchInserterIndexProvider(inserter);
        this.batchInserterIndexes = new HashMap<>();

    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
@Test
public void batchInsert() throws Exception
{
    Neo4jTestCase.deleteFileOrDirectory( new File(
            "target/neo4jdb-batchinsert" ) );
    // START SNIPPET: batchInsert
    BatchInserter inserter = BatchInserters.inserter( "target/neo4jdb-batchinsert" );
    BatchInserterIndexProvider indexProvider =
            new LuceneBatchInserterIndexProvider( inserter );
    BatchInserterIndex actors =
            indexProvider.nodeIndex( "actors", MapUtil.stringMap( "type", "exact" ) );
    actors.setCacheCapacity( "name", 100000 );

    Map<String, Object> properties = MapUtil.map( "name", "Keanu Reeves" );
    long node = inserter.createNode( properties );
    actors.add( node, properties );

    //make the changes visible for reading, use this sparsely, requires IO!
    actors.flush();

    // Make sure to shut down the index provider as well
    indexProvider.shutdown();
    inserter.shutdown();
    // END SNIPPET: batchInsert

    GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabase( "target/neo4jdb-batchinsert" );
    try ( Transaction tx = db.beginTx() )
    {
        Index<Node> index = db.index().forNodes( "actors" );
        Node reeves = index.get( "name", "Keanu Reeves" ).next();
        assertEquals( node, reeves.getId() );
    }
    db.shutdown();
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
public void startServer() throws Exception {
	if (!this.contains("db_dir"))
		throw new Exception("Need to add server directory.");
	else
		System.out.println("Database output dir is: " + this.getSetting("db_dir"));

	if (this.contains("restart_neo4j_with"))
		System.out.println("Will restart neo4j after use." + this.getSetting("restart_neo4j_with"));

	if (this.contains("delete_old_data") && this.getSetting("delete_old_data").equals("true")) {
		System.out.println("Will delete old data.");
		deleteDirectory(new File(this.getSetting("db_dir")));

		new File(this.getSetting("db_dir")).mkdir();
	}

	if (this.contains("restart_neo4j_with")) {
		Process p = Runtime.getRuntime().exec(this.getSetting("restart_neo4j_with") + " stop");
		p.waitFor();
		System.out.println("Stopped Neo4j!");
	}

	// inserter = BatchInserters.inserter(new
	// File(this.getSetting("db_dir")).getAbsolutePath());

	indexProvider = new LuceneBatchInserterIndexProvider(inserter);
	// index = indexProvider.nodeIndex("Node-exact",
	// MapUtil.stringMap("type", "exact"));

	Runtime.getRuntime().addShutdownHook(new Thread() {
		public void run() {
			try {
				shutdown();
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	});
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
public BatchHandler(BatchInserter db2, int indexCache, int timeout) {
	this.db = db2;
	this.indexCache = indexCache;
	this.timeout = timeout;

	BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(db);
	index = indexProvider.nodeIndex("ttlIndex", MapUtil.stringMap("type", "exact"));
	index.setCacheCapacity(URI_PROPERTY, indexCache + 1);

}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
@Inject
public GraphBatchImpl(BatchInserter inserter, @IndicatesUniqueProperty String uniqueProperty,
    @IndicatesIndexedProperties Set<String> indexedProperties,
    @IndicatesExactIndexedProperties Set<String> exactIndexedProperties,
    ConcurrentMap<String, Long> idMap, RelationshipMap relationshioMap) {
  this.inserter = inserter;
  this.idMap = idMap;
  this.relationshipMap = relationshioMap;
  indexProvider = new LuceneBatchInserterIndexProvider(inserter);
  nodeIndex = indexProvider.nodeIndex("node_auto_index", INDEX_CONFIG);
  this.uniqueProperty = uniqueProperty;
  this.indexedProperties = newHashSet(indexedProperties);
  this.indexedProperties.add(uniqueProperty);
  this.indexedExactProperties = newHashSet(exactIndexedProperties);
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
protected Tuple<BatchInserterIndex, BatchInserterIndexProvider> getOrCreateIndex(final String name, final String property,
                                                                                 final boolean nodeIndex, final int cachSize) {

	final BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
	final BatchInserterIndex index;

	if (nodeIndex) {

		index = indexProvider.nodeIndex(name, MapUtil.stringMap("type", "exact"));
	} else {

		index = indexProvider.relationshipIndex(name, MapUtil.stringMap("type", "exact"));
	}

	index.setCacheCapacity(property, cachSize);

	return Tuple.tuple(index, indexProvider);
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
public static void run(final String neo4storage,
        final String redisQueueName, final String redisHost) {

    ObelixQueue redisQueueManager = new RedisObelixQueue(redisQueueName, redisHost);

    Label userLabel = DynamicLabel.label("User");
    Label itemLabel = DynamicLabel.label("Item");

    Map<String, Long> usersNodesMap = new HashMap<>();
    Map<String, Long> itemsNodesMap = new HashMap<>();
    Map<String, Long> relationshipsMap = new HashMap<>();

    Map<String, String> config = new HashMap<>();
    config.put("neostore.nodestore.db.mapped_memory", "2G");
    config.put("neostore.relationshipstore.db.mapped_memory", "9G");
    config.put("neostore.propertystore.db.mapped_memory", "800M");
    config.put("neostore.propertystore.db.strings.mapped_memory", "800M");
    config.put("neostore.propertystore.db.arrays.mapped_memory", "500M");

    BatchInserter inserter;

    inserter = BatchInserters.inserter(neo4storage, config);

    BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);

    registerShutdownHook(inserter, indexProvider);

    BatchInserterIndex usersIndex = indexProvider.nodeIndex("users",
            MapUtil.stringMap("type", "exact"));
    usersIndex.setCacheCapacity("node_id", NEO_CACHE_CAPACITY);

    BatchInserterIndex itemsIndex = indexProvider.nodeIndex("items",
            MapUtil.stringMap("type", "exact"));
    usersIndex.setCacheCapacity("node_id", NEO_CACHE_CAPACITY);

    BatchInserterIndex relationshipIndex = indexProvider.relationshipIndex("relationships",
            MapUtil.stringMap("type", "exact"));
    usersIndex.setCacheCapacity("timestamp", NEO_CACHE_CAPACITY);

    boolean notFinised = true;

    int c = 0;
    while (notFinised) {

        ObelixQueueElement result = redisQueueManager.pop();

        if (result == null) {
            notFinised = false;
        }

        if (result != null) {

            NeoEvent event = getEvent(result.toString());

            if (event == null) {
                continue;
            }

            long userid = getOrCreateUserNodeID(event.getUser(), usersNodesMap,
                    usersIndex, inserter, userLabel);

            long itemid = getOrCreateItemNodeID(event.getItem(), itemsNodesMap,
                    itemsIndex, inserter, itemLabel);


            getOrCreateRelatinshipID(event.getTimestamp(),
                    userid, itemid, relationshipsMap, relationshipIndex,
                    inserter, NeoHelpers.RelTypes.VIEWED);

        }

        c += 1;

        if (c % IMPORTS_BETWEEN_EACH_LOG_MESSAGE == 0) {
            LOGGER.info("Imported " + c);
        }
    }
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
/**
 * Adds Switzerland, cantons, regions and municipalities.
 */
public HashMap<Long, Integer> addGeographicalInfo()
{		
	lStartTime = System.currentTimeMillis();
	
	System.out.println("Adding geographical information...");
	
	inserter = BatchInserters.inserter(Main.dbPath);
	indexProvider = new LuceneBatchInserterIndexProvider(inserter);

	geoIndex = indexProvider.nodeIndex("geoIndex", MapUtil.stringMap("type", "exact"));
	geoIndex.setCacheCapacity("name", 3000);

	addSwitzerland();
	addSwissGeoNodes();
	
	indexProvider.shutdown();
	inserter.shutdown();

	lEndTime = System.currentTimeMillis();

	System.out.println(String.format("Added %d cantons,  %d regions, %d municipalities in %.2f seconds", cantonsAdded, regionsAdded, municipalities.size(), (lEndTime - lStartTime)/1000.0));

	return municipalities;
}
 

import org.neo4j.index.lucene.unsafe.batchinsert.LuceneBatchInserterIndexProvider; //导入依赖的package包/类
private void manageActivities()
{
	lStartTime = System.currentTimeMillis();
	
	System.out.println("Adding activities...");
	
	inserter = BatchInserters.inserter(Main.dbPath);
	indexProvider = new LuceneBatchInserterIndexProvider(inserter);

	activityIndex = indexProvider.nodeIndex("activityIndex", MapUtil.stringMap("type", "exact"));
	activityIndex.setCacheCapacity("name", 1000);

	addActivities();
	
	System.out.println("Shutting down...");
	
	indexProvider.shutdown();
	inserter.shutdown();
	
	lEndTime = System.currentTimeMillis();
	
	System.out.println(String.format("Added %d activities in %.2f seconds", activities.size(), (lEndTime - lStartTime) / 1000.0));
}