Java Codec类代码示例

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
public CodecService(@Nullable MapperService mapperService, Logger logger) {
    final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
    if (mapperService == null) {
        codecs.put(DEFAULT_CODEC, new Lucene62Codec());
        codecs.put(BEST_COMPRESSION_CODEC, new Lucene62Codec(Mode.BEST_COMPRESSION));
    } else {
        codecs.put(DEFAULT_CODEC,
                new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
        codecs.put(BEST_COMPRESSION_CODEC,
                new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
    }
    codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
    for (String codec : Codec.availableCodecs()) {
        codecs.put(codec, Codec.forName(codec));
    }
    this.codecs = codecs.immutableMap();
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
private void assertCompressionEquals(Mode expected, Codec actual) throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(null);
    iwc.setCodec(actual);
    IndexWriter iw = new IndexWriter(dir, iwc);
    iw.addDocument(new Document());
    iw.commit();
    iw.close();
    DirectoryReader ir = DirectoryReader.open(dir);
    SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
    String v = sr.getSegmentInfo().info.getAttribute(Lucene50StoredFieldsFormat.MODE_KEY);
    assertNotNull(v);
    assertEquals(expected, Mode.valueOf(v));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
private CodecService(Index index, Settings indexSettings, MapperService mapperService) {
    super(index, indexSettings);
    this.mapperService = mapperService;
    MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
    if (mapperService == null) {
        codecs.put(DEFAULT_CODEC, new Lucene54Codec());
        codecs.put(BEST_COMPRESSION_CODEC, new Lucene54Codec(Mode.BEST_COMPRESSION));
    } else {
        codecs.put(DEFAULT_CODEC, 
                new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
        codecs.put(BEST_COMPRESSION_CODEC, 
                new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
    }
    codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
    for (String codec : Codec.availableCodecs()) {
        codecs.put(codec, Codec.forName(codec));
    }
    this.codecs = codecs.immutableMap();
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
private IndexWriterConfig newConfig(final Analyzer analyzer) {
    final IndexWriterConfig config = new IndexWriterConfig(analyzer);
    final Codec codec = new Lucene62Codec() {
        @Override
        public PostingsFormat getPostingsFormatForField(String field) {
            if (field.equals("id")) {
                return PostingsFormat.forName("Memory");
            } else {
                return PostingsFormat.forName("Lucene50");
            }
        }
    };
    config.setCodec(codec);

    return config;
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
@Override
protected void addRandomFields(Document doc) {
  for (IndexOptions opts : IndexOptions.values()) {
    final String field = "f_" + opts;
    String pf = TestUtil.getPostingsFormat(Codec.getDefault(), field);
    if (opts == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS && doesntSupportOffsets.contains(pf)) {
      continue;
    }
    FieldType ft = new FieldType();
    ft.setIndexOptions(opts);
    ft.setIndexed(true);
    ft.setOmitNorms(true);
    ft.freeze();
    final int numFields = random().nextInt(5);
    for (int j = 0; j < numFields; ++j) {
      doc.add(new Field("f_" + opts, TestUtil.randomSimpleString(random(), 2), ft));
    }
  }
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
/**
 * Check codec restrictions.
 * 
 * @throws AssumptionViolatedException if the class does not work with a given codec.
 */
private void checkCodecRestrictions(Codec codec) {
  assumeFalse("Class not allowed to use codec: " + codec.getName() + ".",
      shouldAvoidCodec(codec.getName()));

  if (codec instanceof RandomCodec && !avoidCodecs.isEmpty()) {
    for (String name : ((RandomCodec)codec).formatNames) {
      assumeFalse("Class not allowed to use postings format: " + name + ".",
          shouldAvoidCodec(name));
    }
  }

  PostingsFormat pf = codec.postingsFormat();
  assumeFalse("Class not allowed to use postings format: " + pf.getName() + ".",
      shouldAvoidCodec(pf.getName()));

  assumeFalse("Class not allowed to use postings format: " + LuceneTestCase.TEST_POSTINGSFORMAT + ".", 
      shouldAvoidCodec(LuceneTestCase.TEST_POSTINGSFORMAT));
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields, boolean allowPreFlex) throws Throwable {

    final int termIndexInterval = TestUtil.nextInt(random(), 13, 27);
    final Codec codec = Codec.getDefault();
    final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null);
    final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, termIndexInterval, null, newIOContext(random()));

    final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);
    Arrays.sort(fields);
    for (final FieldData field : fields) {
      if (!allowPreFlex && codec instanceof Lucene3xCodec) {
        // code below expects unicode sort order
        continue;
      }
      field.write(consumer);
    }
    consumer.close();
  }
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
public void testReuseDocsEnumNoReuse() throws IOException {
  Directory dir = newDirectory();
  Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
      newIndexWriterConfig(new MockAnalyzer(random())).setCodec(cp));
  int numdocs = atLeast(20);
  createRandomIndex(numdocs, writer, random());
  writer.commit();

  DirectoryReader open = DirectoryReader.open(dir);
  for (AtomicReaderContext ctx : open.leaves()) {
    AtomicReader indexReader = ctx.reader();
    Terms terms = indexReader.terms("body");
    TermsEnum iterator = terms.iterator(null);
    IdentityHashMap<DocsEnum, Boolean> enums = new IdentityHashMap<>();
    MatchNoBits bits = new Bits.MatchNoBits(indexReader.maxDoc());
    while ((iterator.next()) != null) {
      DocsEnum docs = iterator.docs(random().nextBoolean() ? bits : new Bits.MatchNoBits(indexReader.maxDoc()), null, random().nextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
      enums.put(docs, true);
    }
    
    assertEquals(terms.size(), enums.size());
  }
  IOUtils.close(writer, open, dir);
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
public FieldInfos createAndWriteFieldInfos(Directory dir, String filename) throws IOException{
//Positive test of FieldInfos
  assertTrue(testDoc != null);
  FieldInfos.Builder builder = new FieldInfos.Builder();
  for (IndexableField field : testDoc) {
    builder.addOrUpdate(field.name(), field.fieldType());
  }
  FieldInfos fieldInfos = builder.finish();
  //Since the complement is stored as well in the fields map
  assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
  
  
  IndexOutput output = dir.createOutput(filename, newIOContext(random()));
  assertTrue(output != null);
  //Use a RAMOutputStream

  FieldInfosWriter writer = Codec.getDefault().fieldInfosFormat().getFieldInfosWriter();
  writer.write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
  output.close();
  return fieldInfos;
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
/**
 * test collection-level term stats (new in 4.x indexes)
 */
public void testTotalTermFreq() throws Exception {
  assumeFalse("PreFlex codec does not support collection-level term stats", "Lucene3x".equals(Codec.getDefault().getName()));

  clearIndex();
  
  assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
  assertU(adoc("id","2", "a_t","how now brown cow"));
  assertU(commit()); // create more than one segment
  assertU(adoc("id","3", "a_t","brown cow"));
  assertU(adoc("id","4"));
  assertU(commit()); // create more than one segment
  assertU(adoc("id","5"));
  assertU(adoc("id","6", "a_t","cow cow cow cow cow"));
  assertU(commit());
  assertQ(req("fl","*,score","q", "{!func}totaltermfreq('a_t','cow')", "fq","id:6"), "//float[@name='score']='7.0'");    
  assertQ(req("fl","*,score","q", "{!func}ttf(a_t,'cow')", "fq","id:6"), "//float[@name='score']='7.0'");
  assertQ(req("fl","*,score","q", "{!func}sumtotaltermfreq('a_t')", "fq","id:6"), "//float[@name='score']='11.0'");
  assertQ(req("fl","*,score","q", "{!func}sttf(a_t)", "fq","id:6"), "//float[@name='score']='11.0'");
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
private void doTestMixedPostings(Codec codec) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(codec);
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  // turn on vectors for the checkindex cross-check
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  Field idField = new Field("id", "", ft);
  Field dateField = new Field("date", "", ft);
  doc.add(idField);
  doc.add(dateField);
  for (int i = 0; i < 100; i++) {
    idField.setStringValue(Integer.toString(random().nextInt(50)));
    dateField.setStringValue(Integer.toString(random().nextInt(100)));
    iw.addDocument(doc);
  }
  iw.close();
  dir.close(); // checkindex
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
public void testReader() throws IOException {
  TermVectorsReader reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg.info, fieldInfos, newIOContext(random()));
  for (int j = 0; j < 5; j++) {
    Terms vector = reader.get(j).terms(testFields[0]);
    assertNotNull(vector);
    assertEquals(testTerms.length, vector.size());
    TermsEnum termsEnum = vector.iterator(null);
    for (int i = 0; i < testTerms.length; i++) {
      final BytesRef text = termsEnum.next();
      assertNotNull(text);
      String term = text.utf8ToString();
      //System.out.println("Term: " + term);
      assertEquals(testTerms[i], term);
    }
    assertNull(termsEnum.next());
  }
  reader.close();
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
public void testDisableImpersonation() throws Exception {
  Codec[] oldCodecs = new Codec[] { new Lucene40RWCodec(), new Lucene41RWCodec(), new Lucene42RWCodec() };
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setCodec(oldCodecs[random().nextInt(oldCodecs.length)]);
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "bar", Store.YES));
  doc.add(new NumericDocValuesField("n", 18L));
  writer.addDocument(doc);
  
  OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;
  try {
    writer.close();
    fail("should not have succeeded to impersonate an old format!");
  } catch (UnsupportedOperationException e) {
    writer.rollback();
  } finally {
    OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
  }
  
  dir.close();
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
/**
 * Reloads all Lucene SPI implementations using the new classloader.
 * This method must be called after the new classloader has been created to
 * register the services for use.
 */
static void reloadLuceneSPI(ClassLoader loader) {
    // do NOT change the order of these method calls!

    // Codecs:
    PostingsFormat.reloadPostingsFormats(loader);
    DocValuesFormat.reloadDocValuesFormats(loader);
    Codec.reloadCodecs(loader);
    // Analysis:
    CharFilterFactory.reloadCharFilters(loader);
    TokenFilterFactory.reloadTokenFilters(loader);
    TokenizerFactory.reloadTokenizers(loader);
}
 

import org.apache.lucene.codecs.Codec; //导入依赖的package包/类
/**
 * Construct a new complete SegmentInfo instance from input.
 * <p>Note: this is public only to allow access from
 * the codecs package.</p>
 */
public SegmentInfo(Directory dir, Version version, String name, int docCount,
                   boolean isCompoundFile, Codec codec, Map<String,String> diagnostics, Map<String,String> attributes) {
  assert !(dir instanceof TrackingDirectoryWrapper);
  this.dir = dir;
  this.version = version;
  this.name = name;
  this.docCount = docCount;
  this.isCompoundFile = isCompoundFile;
  this.codec = codec;
  this.diagnostics = diagnostics;
  this.attributes = attributes;
}