Java Tokenizer.tokenize方法代码示例

import org.lemurproject.galago.core.tokenize.Tokenizer; //导入方法依赖的package包/类
@Override
public Document fromStream(DataInputStream stream, Document.DocumentComponents selection) throws IOException {
  SerializerCommon.ByteBuf buffer = new SerializerCommon.ByteBuf();
  DataInputStream input = new DataInputStream(new SnappyInputStream(stream));
  Document d = new Document();

  int metadataSize = input.readInt();
  int textSize = input.readInt(); // ignored

  // identifier
  d.identifier = input.readLong();

  // name
  d.name = buffer.readString(input);

  if (selection.metadata) {
    d.metadata = SerializerCommon.readMetadata(input, buffer);
    // only both skipping if we need to
  } else if (selection.text || selection.tokenize) {
    input.skip(metadataSize);
  }

  // can't get tokens without text in this case...
  if (selection.text || selection.tokenize) {
    d.text = SerializerCommon.readText(input, selection, buffer);
  }
  input.close();

  // give back terms & tags
  if(selection.tokenize) {
    // Tokenizer is *not* threadsafe, so we must make a copy of it for each use in case of threads.
    Tokenizer tokenizer = getTokenizer();
    tokenizer.tokenize(d);
  }

  return d;
}
 

import org.lemurproject.galago.core.tokenize.Tokenizer; //导入方法依赖的package包/类
@Override
public void run(Parameters p, PrintStream output) throws Exception {
  Retrieval index = RetrievalFactory.create(p);
  Tokenizer tokenizer = index.getTokenizer();

  String text = StreamUtil.copyStreamToString(StreamCreator.openInputStream(p.getString("input")));

  Document doc = tokenizer.tokenize(text);
  HashSet<String> uniq = new HashSet<>(doc.terms);

  List<Parameters> termInfos = new ArrayList<>();
  for (String query : uniq) {
    Parameters termStats = Parameters.create();
    NodeStatistics counts = index.getNodeStatistics(new Node("counts", query));
    termStats.set("term", query);
    termStats.set("cf", counts.nodeFrequency);
    termStats.set("maxTF", counts.maximumCount);
    termStats.set("df", counts.nodeDocumentCount);
    termInfos.add(termStats);
  }

  Parameters overall = Parameters.create();
  FieldStatistics lengths = index.getCollectionStatistics(new Node("lengths"));
  overall.put("clen", lengths.collectionLength);
  overall.put("terms", termInfos);

  if(p.get("pretty", true)) {
    output.println(overall.toPrettyString());
  } else {
    output.println(overall);
  }
}
 

import org.lemurproject.galago.core.tokenize.Tokenizer; //导入方法依赖的package包/类
@Test
public void testMergeFlushedSequentialIndexes() throws Exception {
    File index1 = null;
    File index2 = null;
    File indexmerged = null;
    try {
        Parameters p1 = Parameters.parseString("{\"documentNumberOffset\":0}");
        Parameters p2 = Parameters.parseString("{\"documentNumberOffset\":1000}");
        MemoryIndex mi1 = new MemoryIndex(new FakeParameters(p1));
        MemoryIndex mi2 = new MemoryIndex(new FakeParameters(p2));

        Tokenizer tok = Tokenizer.create(p1);
        for (int i = 0; i < 100; i++) {
            Document d1 = new Document("DOCS1-" + i, "this is sample document " + i);
            Document d2 = new Document("DOCS2-" + i, "this is a different document " + i);
            tok.tokenize(d1);
            tok.tokenize(d2);
            mi1.process(d1);
            mi2.process(d2);
        }

        index1 = FileUtility.createTemporaryDirectory();
        FlushToDisk.flushMemoryIndex(mi1, index1.getAbsolutePath());

        index2 = FileUtility.createTemporaryDirectory();
        FlushToDisk.flushMemoryIndex(mi2, index2.getAbsolutePath());

        AppTest.verifyIndexStructures(index1);
        AppTest.verifyIndexStructures(index2);

        indexmerged = FileUtility.createTemporaryDirectory();

        Parameters mergeParams = Parameters.create();
        mergeParams.set("indexPath", indexmerged.getAbsolutePath());
        ArrayList<String> inputs = new ArrayList<String>();
        inputs.add(index1.getAbsolutePath());
        inputs.add(index2.getAbsolutePath());
        mergeParams.set("inputPath", inputs);
        mergeParams.set("renumberDocuments", false);
        App.run("merge-index", mergeParams, System.out);

        AppTest.verifyIndexStructures(indexmerged);

        DiskIndex di_index1 = new DiskIndex(index1.getAbsolutePath());
        DiskIndex di_index2 = new DiskIndex(index2.getAbsolutePath());
        DiskIndex di_merged = new DiskIndex(indexmerged.getAbsolutePath());

        assertEquals(di_index1.getIndexPartStatistics("postings").collectionLength, 500);
        assertEquals(di_index2.getIndexPartStatistics("postings").collectionLength, 600);
        assertEquals(di_merged.getIndexPartStatistics("postings").collectionLength, 1100);

        assertEquals(di_index1.getIndexPartStatistics("postings").vocabCount, 104);
        assertEquals(di_index2.getIndexPartStatistics("postings").vocabCount, 105);
        assertEquals(di_merged.getIndexPartStatistics("postings").vocabCount, 106);

        assertEquals(di_index1.getIndexPartStatistics("postings").highestDocumentCount, 100);
        assertEquals(di_index2.getIndexPartStatistics("postings").highestDocumentCount, 100);
        assertEquals(di_merged.getIndexPartStatistics("postings").highestDocumentCount, 200);

        assertEquals(di_index1.getIndexPartStatistics("postings").highestFrequency, 100);
        assertEquals(di_index2.getIndexPartStatistics("postings").highestFrequency, 100);
        assertEquals(di_merged.getIndexPartStatistics("postings").highestFrequency, 200);

        assertEquals(di_merged.getName(50), mi1.getName(50));
        assertEquals(di_merged.getName(1050), mi2.getName(1050));

    } finally {
        if (index1 != null) {
            FSUtil.deleteDirectory(index1);
        }
        if (index2 != null) {
            FSUtil.deleteDirectory(index2);
        }
        if (indexmerged != null) {
            FSUtil.deleteDirectory(indexmerged);
        }
    }
}
 

import org.lemurproject.galago.core.tokenize.Tokenizer; //导入方法依赖的package包/类
public List<String> getTerms(Parameters config) {
  Tokenizer tok = Tokenizer.instance(config);
  List<String> terms = tok.tokenize(text.replace('\u2013', '-')).terms;
  return QueryUtil.filterTerms(config, terms);
}