Java IntsRef类代码示例

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
@Override
public IntsRef subtract(IntsRef output, IntsRef inc) {
  assert output != null;
  assert inc != null;
  if (inc == NO_OUTPUT) {
    // no prefix removed
    return output;
  } else if (inc.length == output.length) {
    // entire output removed
    return NO_OUTPUT;
  } else {
    assert inc.length < output.length: "inc.length=" + inc.length + " vs output.length=" + output.length;
    assert inc.length > 0;
    return new IntsRef(output.ints, output.offset + inc.length, output.length-inc.length);
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
@Override
public IntsRef add(IntsRef prefix, IntsRef output) {
  assert prefix != null;
  assert output != null;
  if (prefix == NO_OUTPUT) {
    return output;
  } else if (output == NO_OUTPUT) {
    return prefix;
  } else {
    assert prefix.length > 0;
    assert output.length > 0;
    IntsRef result = new IntsRef(prefix.length + output.length);
    System.arraycopy(prefix.ints, prefix.offset, result.ints, 0, prefix.length);
    System.arraycopy(output.ints, output.offset, result.ints, prefix.length, output.length);
    result.length = prefix.length + output.length;
    return result;
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
/** Looks up the output for this input, or null if the
 *  input is not accepted. */
public static<T> T get(FST<T> fst, IntsRef input) throws IOException {

  // TODO: would be nice not to alloc this on every lookup
  final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>());

  final BytesReader fstReader = fst.getBytesReader();

  // Accumulate output as we go
  T output = fst.outputs.getNoOutput();
  for(int i=0;i<input.length;i++) {
    if (fst.findTargetArc(input.ints[input.offset + i], arc, arc, fstReader) == null) {
      return null;
    }
    output = fst.outputs.add(output, arc.output);
  }

  if (arc.isFinal()) {
    return fst.outputs.add(output, arc.nextFinalOutput);
  } else {
    return null;
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
private void testRandomWords(int maxNumWords, int numIter) throws IOException {
  Random random = new Random(random().nextLong());
  for(int iter=0;iter<numIter;iter++) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter " + iter);
    }
    for(int inputMode=0;inputMode<2;inputMode++) {
      final int numWords = random.nextInt(maxNumWords+1);
      Set<IntsRef> termsSet = new HashSet<>();
      IntsRef[] terms = new IntsRef[numWords];
      while(termsSet.size() < numWords) {
        final String term = getRandomString(random);
        termsSet.add(toIntsRef(term, inputMode));
      }
      doTest(inputMode, termsSet.toArray(new IntsRef[termsSet.size()]));
    }
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
private final void count(List<MatchingDocs> matchingDocs) throws IOException {
  IntsRef scratch  = new IntsRef();
  for(MatchingDocs hits : matchingDocs) {
    OrdinalsReader.OrdinalsSegmentReader ords = ordinalsReader.getReader(hits.context);
    DocIdSetIterator docs = hits.bits.iterator();
    
    int doc;
    while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
      ords.get(doc, scratch);
      for(int i=0;i<scratch.length;i++) {
        values[scratch.ints[scratch.offset+i]]++;
      }
    }
  }

  rollup();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
@Override
public OrdinalsSegmentReader getReader(AtomicReaderContext context) throws IOException {
  BinaryDocValues values0 = context.reader().getBinaryDocValues(field);
  if (values0 == null) {
    values0 = DocValues.emptyBinary();
  }

  final BinaryDocValues values = values0;

  return new OrdinalsSegmentReader() {
    @Override
    public void get(int docID, IntsRef ordinals) throws IOException {
      final BytesRef bytes = values.get(docID);
      decode(bytes, ordinals);
    }
  };
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
public void testSimpleDictionary() throws Exception {
  InputStream affixStream = getClass().getResourceAsStream("simple.aff");
  InputStream dictStream = getClass().getResourceAsStream("simple.dic");

  Dictionary dictionary = new Dictionary(affixStream, dictStream);
  assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
  assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
  IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
  assertNotNull(ordList);
  assertEquals(1, ordList.length);
  
  BytesRef ref = new BytesRef();
  dictionary.flagLookup.get(ordList.ints[0], ref);
  char flags[] = Dictionary.decodeFlags(ref);
  assertEquals(1, flags.length);
  
  ordList = dictionary.lookupWord(new char[]{'l', 'u', 'c', 'e', 'n'}, 0, 5);
  assertNotNull(ordList);
  assertEquals(1, ordList.length);
  dictionary.flagLookup.get(ordList.ints[0], ref);
  flags = Dictionary.decodeFlags(ref);
  assertEquals(1, flags.length);
  
  affixStream.close();
  dictStream.close();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
public void testCompressedDictionary() throws Exception {
  InputStream affixStream = getClass().getResourceAsStream("compressed.aff");
  InputStream dictStream = getClass().getResourceAsStream("compressed.dic");

  Dictionary dictionary = new Dictionary(affixStream, dictStream);
  assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
  assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
  IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
  BytesRef ref = new BytesRef();
  dictionary.flagLookup.get(ordList.ints[0], ref);
  char flags[] = Dictionary.decodeFlags(ref);
  assertEquals(1, flags.length);
  
  affixStream.close();
  dictStream.close();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
public void testCompressedBeforeSetDictionary() throws Exception {
  InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
  InputStream dictStream = getClass().getResourceAsStream("compressed.dic");

  Dictionary dictionary = new Dictionary(affixStream, dictStream);
  assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
  assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
  IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
  BytesRef ref = new BytesRef();
  dictionary.flagLookup.get(ordList.ints[0], ref);
  char flags[] = Dictionary.decodeFlags(ref);
  assertEquals(1, flags.length);
  
  affixStream.close();
  dictStream.close();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
public void testCompressedEmptyAliasDictionary() throws Exception {
  InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
  InputStream dictStream = getClass().getResourceAsStream("compressed.dic");

  Dictionary dictionary = new Dictionary(affixStream, dictStream);
  assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
  assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
  IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
  BytesRef ref = new BytesRef();
  dictionary.flagLookup.get(ordList.ints[0], ref);
  char flags[] = Dictionary.decodeFlags(ref);
  assertEquals(1, flags.length);
  
  affixStream.close();
  dictStream.close();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
/** Adds all leaving arcs, including 'finished' arc, if
 *  the node is final, from this node into the queue.  */
public void addStartPaths(FST.Arc<T> node, T startOutput, boolean allowEmptyString, IntsRef input) throws IOException {

  // De-dup NO_OUTPUT since it must be a singleton:
  if (startOutput.equals(fst.outputs.getNoOutput())) {
    startOutput = fst.outputs.getNoOutput();
  }

  FSTPath<T> path = new FSTPath<T>(startOutput, node, input);
  fst.readFirstTargetArc(node, path.arc, bytesReader);

  //System.out.println("add start paths");

  // Bootstrap: find the min starting arc
  while (true) {
    if (allowEmptyString || path.arc.label != FST.END_LABEL) {
      addIfCompetitive(path);
    }
    if (path.arc.isLast()) {
      break;
    }
    fst.readNextArc(path.arc, bytesReader);
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
private void testRandomWords(int maxNumWords, int numIter) throws IOException {
  Random random = new Random(random().nextLong());
  for(int iter=0;iter<numIter;iter++) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter " + iter);
    }
    for(int inputMode=0;inputMode<2;inputMode++) {
      final int numWords = random.nextInt(maxNumWords+1);
      Set<IntsRef> termsSet = new HashSet<IntsRef>();
      IntsRef[] terms = new IntsRef[numWords];
      while(termsSet.size() < numWords) {
        final String term = getRandomString(random);
        termsSet.add(toIntsRef(term, inputMode));
      }
      doTest(inputMode, termsSet.toArray(new IntsRef[termsSet.size()]));
    }
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final Builder<Object> builder = new Builder<Object>(
      FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
      shareMaxTailLength, outputs, null, false, 
      PackedInts.DEFAULT, true, 15);
  
  BytesRef scratch = new BytesRef();
  BytesRef entry;
  final IntsRef scratchIntsRef = new IntsRef();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.compareTo(entry) != 0) {
      builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : builder.finish();
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
@Override
public void encode(IntsRef values, BytesRef buf) {
  buf.offset = buf.length = 0;
  int upto = values.offset + values.length;
  for (int i = values.offset; i < upto; i++) {
    int value = values.ints[i];
    if (value == 1) {
      indicator |= ENCODE_TABLE[ordinal];
    } else {
      encodeQueue.ints[encodeQueue.length++] = value - 2;
    }
    ++ordinal;
    
    // encode the chunk and the indicator
    if (ordinal == 8) {
      encodeChunk(buf);
    }
  }
  
  // encode remaining values
  if (ordinal != 0) {
    encodeChunk(buf);
  }
}
 

import org.apache.lucene.util.IntsRef; //导入依赖的package包/类
@BeforeClass
public static void beforeClassEncodingTest() throws Exception {
  int capacity = atLeast(10000);
  data = new IntsRef(capacity);
  for (int i = 0; i < 10; i++) {
    data.ints[i] = i + 1; // small values
  }
  for (int i = 10; i < data.ints.length; i++) {
    data.ints[i] = random().nextInt(Integer.MAX_VALUE - 1) + 1; // some encoders don't allow 0
  }
  data.length = data.ints.length;
  
  uniqueSortedData = IntsRef.deepCopyOf(data);
  Arrays.sort(uniqueSortedData.ints);
  uniqueSortedData.length = 0;
  int prev = -1;
  for (int i = 0; i < uniqueSortedData.ints.length; i++) {
    if (uniqueSortedData.ints[i] != prev) {
      uniqueSortedData.ints[uniqueSortedData.length++] = uniqueSortedData.ints[i];
      prev = uniqueSortedData.ints[i];
    }
  }
}