Java MMapDirectory类代码示例

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,
        Set<String> preLoadExtensions) throws IOException {
    if (preLoadExtensions.isEmpty() == false
            && directory instanceof MMapDirectory
            && ((MMapDirectory) directory).getPreload() == false) {
        if (preLoadExtensions.contains("*")) {
            ((MMapDirectory) directory).setPreload(true);
            return directory;
        }
        MMapDirectory primary = new MMapDirectory(location, lockFactory);
        primary.setPreload(true);
        return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) {
            @Override
            public String[] listAll() throws IOException {
                // avoid listing twice
                return primary.listAll();
            }
        };
    }
    return directory;
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
	 * @param args
	 * @throws IOException
	 */
	public static void main(String[] args) throws IOException {
		if (args.length != 2) {
			System.err
					.println("Incorrect number of arguments; expected <index_dir> <query_number>");
		}
		String indexDir = args[0];
		int i = Integer.parseInt(args[1]);
		Queries queries = new Queries();
		Directory directory = MMapDirectory.open(new File(indexDir));
		DirectoryReader reader = DirectoryReader.open(directory);
		TermFreqAnalyzer termFreqAnalyzer = new TermFreqAnalyzer(reader,
				queries.getQuery(i));
		termFreqAnalyzer.printAvgTf();
		termFreqAnalyzer.printAvgTfInDocsWithAllTerms();
		reader.close();
//		directory.close();
	}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
 * Constructor.
 *
 * @param luceneIndexPath  Path to the lucene index files
 * @param maxResults  Maximum number of allowed results in a page
 * @param searchTimeout  Maximum time in milliseconds that a lucene search can run
 */
public LuceneSearchProvider(String luceneIndexPath, int maxResults, int searchTimeout) {
    this.luceneIndexPath = luceneIndexPath;
    Utils.createParentDirectories(this.luceneIndexPath);

    this.maxResults = maxResults;
    this.searchTimeout = searchTimeout;

    try {
        luceneDirectory = new MMapDirectory(Paths.get(this.luceneIndexPath));
        luceneIndexIsHealthy = true;
    } catch (IOException e) {
        luceneIndexIsHealthy = false;
        String message = ErrorMessageFormat.UNABLE_TO_CREATE_DIR.format(this.luceneIndexPath);
        LOG.error(message, e);
    }
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
 * Construct a new indexer object.
 * 
 * @param prop
 *            A {@link Properties} object.
 * @throws IOException
 */
public Indexer (Properties prop) throws IOException {
    if (path == null) {
        path = prop.getProperty("krill.indexDir");
    }

    log.info("Output directory: " + path);

    // Default to 1000 documents till the next commit
    String commitCount = prop.getProperty("krill.index.commit.count",
            "1000");

    // Create a new index object based on the directory
    this.index = new KrillIndex(new MMapDirectory(Paths.get(path)));
    this.count = 0;
    this.commitCount = Integer.parseInt(commitCount);

    jsonFilePattern = Pattern.compile(".*\\.json\\.gz$");
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
@Test
public void advancedTest() throws IOException {
	File indexFile = new File("data", "test3");
	FSDirectory directory = MMapDirectory.open(indexFile.toPath());

	LuceneObjectKVS<Integer, Sample> kvs = new LuceneObjectKVS<>(directory, indexFile, true);

	Sample sample = new Sample();
	sample.name = "hoge";
	sample.age = 10;

	kvs.put(1, sample);
	kvs.put(2, sample);

	// get object
	Sample actual = kvs.get(1);
	assertEquals(sample.name, actual.name);

	// iterator
	Iterator<Entry<Integer, Sample>> ite = kvs.iterator();
	while(ite.hasNext()) {
		ite.next();
	}

	kvs.close();
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
private void initialize(File indexPath, int kmerSize, int kmerSkips, boolean minStrandKmer, double minShouldMatch, QueryGenerationAlgorithm queryGenerationAlgorithm, Similarity similarity) throws Exception {
    if(!indexPath.exists() || !indexPath.isDirectory()) {
        throw new IllegalArgumentException("indexPath is not a directory or does not exist");
    }
    
    this.indexPath = indexPath;
    this.kmerSize = kmerSize;
    this.kmerSkips = kmerSkips;
    this.minStrandKmer = minStrandKmer;
    this.queryAnalyzer = new KmerQueryAnalyzer(this.kmerSize, this.kmerSkips, this.minStrandKmer);
    Directory dir = new MMapDirectory(this.indexPath.toPath()); 
    this.indexReader = DirectoryReader.open(dir);
    this.indexSearcher = new IndexSearcher(this.indexReader);
    if(similarity != null) {
        this.indexSearcher.setSimilarity(similarity);
    }
    this.minShouldMatch = minShouldMatch;
    this.queryGenerationAlgorithm = queryGenerationAlgorithm;
    
    BooleanQuery.setMaxClauseCount(10000);
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
 * Create a (possibly) shared NLP environment. The given data directory
 * must be created (usually from a downloaded zipfile, check the README).
 * Expect many open files and many reads. Network filesystems are known to
 * perform poorly as data directories. Strive to use a local directory if
 * possible, or at least the Lucene indices otherwise.
 * 
 * config.properties can be either in the data directory or the working
 * directory. This is to allow sharing (read-only) indices while still
 * allowing separate development configurations.  
 */
public Environment() {
	
	// Now do some per-thread setup
	db = new Database(this);
	rdf = TDBFactory.assembleDataset(
			pathMustExist("rdf/jena-lucene.ttl"));
	
	// Lucene indexes have huge overhead so avoid re-instantiating by putting them in the Environment
	IndexReader reader;
	try {
		reader = DirectoryReader.open(new MMapDirectory(Paths.get(getConfOrDie("lucene_index"))));
	} catch (IOException e) {
		e.printStackTrace();
		throw new RuntimeException("The candidate-answer Lucene index failed to open.");
	}
	lucene = new IndexSearcher(reader);
	//lucene.setSimilarity(new BM25Similarity());
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
public void testSearchBenchmark() throws IOException {

        System.out.printf("%s\t%s\t%s\t%s\t%s\t%s\n", "Test", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]");
        IndexReader reader = DirectoryReader.open(MMapDirectory.open(new File(indexPath)));
        getRecall("ColorLayout (linear)", new GenericFastImageSearcher(50, ColorLayout.class, DocumentBuilder.FIELD_NAME_COLORLAYOUT, true, reader), reader);
        getRecall("PHOG (linear)", new GenericFastImageSearcher(50, PHOG.class, DocumentBuilder.FIELD_NAME_PHOG, true, reader), reader);
        getRecall("JCD (linear)", new GenericFastImageSearcher(50, JCD.class, DocumentBuilder.FIELD_NAME_JCD, true, reader), reader);
        getRecall("EdgeHistogram (linear)", new GenericFastImageSearcher(50, EdgeHistogram.class, DocumentBuilder.FIELD_NAME_EDGEHISTOGRAM, true, reader), reader);
//        getRecall("CEDD (linear)", new GenericFastImageSearcher(50, CEDD.class, DocumentBuilder.FIELD_NAME_CEDD, true, reader), reader);
//        getRecall("JointHistogram (linear)", new GenericFastImageSearcher(50, JointHistogram.class, DocumentBuilder.FIELD_NAME_JOINT_HISTOGRAM, true, reader), reader);
//        getRecall("LocalBinaryPatterns (linear)", new GenericFastImageSearcher(50, LocalBinaryPatterns.class, DocumentBuilder.FIELD_NAME_LOCAL_BINARY_PATTERNS, true, reader), reader);
//        getRecall("Luminance Layout (linear)", new GenericFastImageSearcher(50, LuminanceLayout.class, DocumentBuilder.FIELD_NAME_LUMINANCE_LAYOUT, true, reader), reader);
//        getRecall("BinaryPatternsPyramid (linear)", new GenericFastImageSearcher(50, BinaryPatternsPyramid.class, DocumentBuilder.FIELD_NAME_BINARY_PATTERNS_PYRAMID, true, reader), reader);
//
        getRecall("ColorLayout (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_COLORLAYOUT, DocumentBuilder.FIELD_NAME_COLORLAYOUT + "_hash", new ColorLayout(), 1000), reader);
        getRecall("PHOG (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_PHOG, DocumentBuilder.FIELD_NAME_PHOG + "_hash", new PHOG(), 1000), reader);
        getRecall("JCD (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_JCD, DocumentBuilder.FIELD_NAME_JCD + "_hash", new JCD(), 1000), reader);
        getRecall("EdgeHistogram (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_EDGEHISTOGRAM, DocumentBuilder.FIELD_NAME_EDGEHISTOGRAM + "_hash", new EdgeHistogram(), 1000), reader);
//        getRecall("CEDD (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_CEDD, DocumentBuilder.FIELD_NAME_CEDD + "_hash", new JCD(), 1000), reader);
//        getRecall("JointHistogram (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_JOINT_HISTOGRAM, DocumentBuilder.FIELD_NAME_JOINT_HISTOGRAM + "_hash", new JointHistogram(), 1000), reader);
//        getRecall("LocalBinaryPatterns (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_LOCAL_BINARY_PATTERNS, DocumentBuilder.FIELD_NAME_LOCAL_BINARY_PATTERNS + "_hash", new LocalBinaryPatterns(), 1000), reader);
//        getRecall("Luminance Layout (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_LUMINANCE_LAYOUT, DocumentBuilder.FIELD_NAME_LUMINANCE_LAYOUT + "_hash", new LuminanceLayout(), 1000), reader);
//        getRecall("BinaryPatternsPyramid (hashed)", new BitSamplingImageSearcher(50, DocumentBuilder.FIELD_NAME_BINARY_PATTERNS_PYRAMID, DocumentBuilder.FIELD_NAME_BINARY_PATTERNS_PYRAMID + "_hash", new LocalBinaryPatterns(), 1000), reader);
//        getRecall("VLAD (linear)", new GenericFastImageSearcher(1000, GenericByteLireFeature.class, DocumentBuilder.FIELD_NAME_SURF_VLAD, true, reader), reader);
    }
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
@Override
public void init(Properties config) {
    try {
        decoder = new JSONString();
        String indexPath = config.getProperty("subscription.lucene.directory");
        LuceneQueryCache queryCache = new LuceneQueryCache("text");
        Presearcher presearcher = new TermFilteredPresearcher();
        if (indexPath == null || indexPath.isEmpty()) {
            monitor = new Monitor(queryCache, presearcher, new RAMDirectory());
        } else {
            monitor = new Monitor(queryCache, presearcher, new MMapDirectory(new File(indexPath)));
        }
    } catch(IOException e) {
        throw new RuntimeException(e.getCause());
    }
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
public DirectoryTaxonomyWriter getTaxoWriter(int segmentNumber) throws IOException {

		Directory d;

		if (indexConfig.getIndexSettings().getStoreIndexOnDisk()) {
			d = MMapDirectory.open(getPathForFacetsIndex(segmentNumber));
		}
		else {
			String indexSegmentDbName = getIndexSegmentDbName(segmentNumber);
			String indexSegmentCollectionName = getIndexSegmentCollectionName(segmentNumber) + "_facets";
			MongoDirectory mongoDirectory = new MongoDirectory(mongo, indexSegmentDbName, indexSegmentCollectionName, clusterConfig.isSharded(),
					clusterConfig.getIndexBlockSize());
			d = new DistributedDirectory(mongoDirectory);
		}

		NRTCachingDirectory nrtCachingDirectory = new NRTCachingDirectory(d, 2, 10);

		return new DirectoryTaxonomyWriter(nrtCachingDirectory);
	}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
 * Extracts field information from the Lucene index at the given location.
 * </p><p>
 * Important: The {@link DVConfig#numericType} is always set to LONG as the auto-detection code
 * has not been implemented.
 * @param indexLocation where the index is stored.
 * @return a complete list of DocValues-relevant information on the fields contained in the index.
 * @throws IOException if the information could not be extracted.
 */
public static List<DVConfig> getDVConfigs(File indexLocation) throws IOException {
    try (IndexReader reader = DirectoryReader.open(MMapDirectory.open(indexLocation))) {
        Map<String, DVConfig> dvConfigs = new HashMap<>();
        for (AtomicReaderContext context : reader.leaves()) {
            for (FieldInfo fieldInfo : context.reader().getFieldInfos()) {
                if (dvConfigs.containsKey(fieldInfo.name)) {
                    continue;
                }
                String first = getFirst(context.reader(), fieldInfo.name);
                dvConfigs.put(fieldInfo.name, new DVConfig(
                        fieldInfo,
                        fieldInfo.hasDocValues() && fieldInfo.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC
                        ? FieldType.NumericType.LONG : null,
                        first));
            }
        }
        List<DVConfig> configs = new ArrayList<>(dvConfigs.values());
        Collections.sort(configs);
        return configs;
    }
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
public TripleIndex() throws IOException {
	Properties prop = new Properties();
	InputStream input = TripleIndex.class.getResourceAsStream("/config/agdistis.properties");
	prop.load(input);

	String envIndex = System.getenv("AGDISTIS_INDEX");
	String index = envIndex != null ? envIndex : prop.getProperty("index");
	log.info("The index will be here: " + index);

	directory = new MMapDirectory(new File(index));
	ireader = DirectoryReader.open(directory);
	isearcher = new IndexSearcher(ireader);
	this.urlValidator = new UrlValidator();

	cache = CacheBuilder.newBuilder().maximumSize(50000).build();
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
public TripleIndexContext() throws IOException {
	Properties prop = new Properties();
	InputStream input = TripleIndexContext.class.getResourceAsStream("/config/agdistis.properties");
	prop.load(input);

	String envIndex = System.getenv("AGDISTIS_INDEX_2");
	String index = envIndex != null ? envIndex : prop.getProperty("index2");
	log.info("The index will be here: " + index);

	directory = new MMapDirectory(new File(index));
	ireader = DirectoryReader.open(directory);
	isearcher = new IndexSearcher(ireader);
	new UrlValidator();

	cache = CacheBuilder.newBuilder().maximumSize(50000).build();
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
public void initSettings(Path stateDir, LuceneSettings settings) throws Exception {
    if (this.settings != null)
        throw new Exception("Init settings is only allowed once");
    this.settings = settings;

    MMapDirectory indexDirectory = new MMapDirectory(stateDir.resolve("index"));
    indexDirectory.setUseUnmap(false);

    MMapDirectory taxoDirectory = new MMapDirectory(stateDir.resolve("taxo"));
    taxoDirectory.setUseUnmap(false);

    IndexWriterConfig config = new IndexWriterConfig(settings.analyzer);
    config.setSimilarity(settings.similarity);
    config.setMergePolicy(settings.getMergePolicy());

    this.indexWriter = new IndexWriter(indexDirectory, config);
    this.indexWriter.commit();
    this.taxoWriter = new DirectoryTaxonomyWriter(taxoDirectory, IndexWriterConfig.OpenMode.CREATE_OR_APPEND, new LruTaxonomyWriterCache(settings.lruTaxonomyWriterCacheSize));
    this.taxoWriter.commit();

    this.scoreCollectorCache = Collections.synchronizedMap(new LRUMap<KeyNameQuery, ScoreSuperCollector>(50));
    this.keyCollectorCache = Collections.synchronizedMap(new LRUMap<KeyNameQuery, FixedBitSet>(50));

    this.manager = new SearcherTaxonomyManager(indexDirectory, taxoDirectory, new MerescoSearchFactory(indexDirectory, taxoDirectory, settings));
    this.manager.addListener(refreshListener);
}
 

import org.apache.lucene.store.MMapDirectory; //导入依赖的package包/类
/**
 * Creates a new {@link MMapDirectory} for the {@link #location} and sets it to {@link #directory}.
 * If an {@link IOException} during creation will be thrown it will be logged and the old
 * {@link Directory} will be kept.
 */
private void createDirectory() {
	try {
		directory = new MMapDirectory(location);
	} catch (final IOException e) {
		LOGGER.error("Exception was thrown during instantiaton of MMapDirectory for location {}.",
				location.getAbsolutePath(), e);
	}
}