Java Util.toUTF32方法代码示例

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
private FST<IntsRef> affixFST(TreeMap<String,List<Integer>> affixes) throws IOException {
  IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton();
  Builder<IntsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, outputs);
  IntsRefBuilder scratch = new IntsRefBuilder();
  for (Map.Entry<String,List<Integer>> entry : affixes.entrySet()) {
    Util.toUTF32(entry.getKey(), scratch);
    List<Integer> entries = entry.getValue();
    IntsRef output = new IntsRef(entries.size());
    for (Integer c : entries) {
      output.ints[output.length++] = c;
    }
    builder.add(scratch.get(), output);
  }
  return builder.finish();
}
 

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
public void testFiniteStringsEatsStack() {
  char[] chars = new char[50000];
  TestUtil.randomFixedLengthUnicodeString(random(), chars, 0, chars.length);
  String bigString1 = new String(chars);
  TestUtil.randomFixedLengthUnicodeString(random(), chars, 0, chars.length);
  String bigString2 = new String(chars);
  Automaton a = Operations.union(Automata.makeString(bigString1), Automata.makeString(bigString2));
  Set<IntsRef> strings = getFiniteStrings(a, -1, false);
  assertEquals(2, strings.size());
  IntsRefBuilder scratch = new IntsRefBuilder();
  Util.toUTF32(bigString1.toCharArray(), 0, bigString1.length(), scratch);
  assertTrue(strings.contains(scratch.get()));
  Util.toUTF32(bigString2.toCharArray(), 0, bigString2.length(), scratch);
  assertTrue(strings.contains(scratch.get()));
}
 

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
public void testSingletonNoLimit() {
  Set<IntsRef> result = Operations.getFiniteStrings(Automata.makeString("foobar"), -1);
  assertEquals(1, result.size());
  IntsRefBuilder scratch = new IntsRefBuilder();
  Util.toUTF32("foobar".toCharArray(), 0, 6, scratch);
  assertTrue(result.contains(scratch.get()));
}
 

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
public void testSingletonLimit1() {
  Set<IntsRef> result = Operations.getFiniteStrings(Automata.makeString("foobar"), 1);
  assertEquals(1, result.size());
  IntsRefBuilder scratch = new IntsRefBuilder();
  Util.toUTF32("foobar".toCharArray(), 0, 6, scratch);
  assertTrue(result.contains(scratch.get()));
}
 

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
private FST.Arc<Long> matchToken(FST.Arc<Long> asc, CharsRef text) throws IOException {
	IntsRef intsRef = Util.toUTF32(text, scratchInfs);
	Long pendingOutput = asc.output;
	for (int intsRefIndex = 0; intsRefIndex < intsRef.length; ++intsRefIndex) {
		if (fst.findTargetArc(intsRef.ints[intsRef.offset + intsRefIndex], asc, asc, fstReader) == null) {
			return null;
		}
		pendingOutput = fst.outputs.add(pendingOutput, asc.output);
	}
	asc.output = pendingOutput;
	return asc;
}
 

import org.apache.lucene.util.fst.Util; //导入方法依赖的package包/类
public void testRandomFiniteStrings1() {

    int numStrings = atLeast(100);
    if (VERBOSE) {
      System.out.println("TEST: numStrings=" + numStrings);
    }

    Set<IntsRef> strings = new HashSet<IntsRef>();
    List<Automaton> automata = new ArrayList<>();
    IntsRefBuilder scratch = new IntsRefBuilder();
    for(int i=0;i<numStrings;i++) {
      String s = TestUtil.randomSimpleString(random(), 1, 200);
      automata.add(Automata.makeString(s));
      Util.toUTF32(s.toCharArray(), 0, s.length(), scratch);
      strings.add(scratch.toIntsRef());
      if (VERBOSE) {
        System.out.println("  add string=" + s);
      }
    }

    // TODO: we could sometimes use
    // DaciukMihovAutomatonBuilder here

    // TODO: what other random things can we do here...
    Automaton a = Operations.union(automata);
    if (random().nextBoolean()) {
      a = MinimizationOperations.minimize(a, 1000000);
      if (VERBOSE) {
        System.out.println("TEST: a.minimize numStates=" + a.getNumStates());
      }
    } else if (random().nextBoolean()) {
      if (VERBOSE) {
        System.out.println("TEST: a.determinize");
      }
      a = Operations.determinize(a, 1000000);
    } else if (random().nextBoolean()) {
      if (VERBOSE) {
        System.out.println("TEST: a.removeDeadStates");
      }
      a = Operations.removeDeadStates(a);
    }

    Set<IntsRef> actual = getFiniteStrings(a, -1, true);
    if (strings.equals(actual) == false) {
      System.out.println("strings.size()=" + strings.size() + " actual.size=" + actual.size());
      List<IntsRef> x = new ArrayList<>(strings);
      Collections.sort(x);
      List<IntsRef> y = new ArrayList<>(actual);
      Collections.sort(y);
      int end = Math.min(x.size(), y.size());
      for(int i=0;i<end;i++) {
        System.out.println("  i=" + i + " string=" + toString(x.get(i)) + " actual=" + toString(y.get(i)));
      }
      fail("wrong strings found");
    }
  }