Java PayloadAttribute.getPayload方法代码示例

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
void walkTerms(TokenStream ts, String op, String[] terms, String[] tags) throws IOException {
    int i = 0;
    while (ts.incrementToken()) {
        CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
        String word = termAtt.toString();
        if (terms != null) {
            assertEquals(terms[i], word);
        }
        if (tags != null) {
            if (tags[i] != null) {
                PayloadAttribute p = ts.getAttribute(PayloadAttribute.class);
                BytesRef payload = p.getPayload();
                //Arrays.copyOfRange(payload.bytes, payload.offset, payload.offset + payload.length);
                byte[] data = payload.bytes;
                assertEquals(tags[i], (data != null) ? new String(data, "UTF-8") : null);
            }
        }
        i++;
    }
    if (terms != null) {
        assertEquals(terms.length, i);
    }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testEncoder() throws Exception {
  Reader reader = new StringReader("the|0.1 quick|0.1 red|0.1");
  TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
  stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);

  stream.reset();
  while (stream.incrementToken()) {
    PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
    assertNotNull(payAttr);
    byte[] payData = payAttr.getPayload().bytes;
    assertNotNull(payData);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertEquals(0.1f, payFloat, 0.0f);
  }
  stream.end();
  stream.close();
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testDelim() throws Exception {
  Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
  TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
  stream = tokenFilterFactory("DelimitedPayload",
      "encoder", "float",
      "delimiter", "*").create(stream);
  stream.reset();
  while (stream.incrementToken()) {
    PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
    assertNotNull(payAttr);
    byte[] payData = payAttr.getPayload().bytes;
    assertNotNull(payData);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertEquals(0.1f, payFloat, 0.0f);
  }
  stream.end();
  stream.close();
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
void assertTermEquals(String expected, TokenStream stream, byte[] expectPay) throws Exception {
  CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
  PayloadAttribute payloadAtt = stream.getAttribute(PayloadAttribute.class);
  assertTrue(stream.incrementToken());
  assertEquals(expected, termAtt.toString());
  BytesRef payload = payloadAtt.getPayload();
  if (payload != null) {
    assertTrue(payload.length + " does not equal: " + expectPay.length, payload.length == expectPay.length);
    for (int i = 0; i < expectPay.length; i++) {
      assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);

    }
  } else {
    assertTrue("expectPay is not null and it should be", expectPay == null);
  }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
private static void assertTokenInfos(TokenStream ts, TokenInfo... infos) throws IOException {
    ts.reset();
    final CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
    final PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
    final PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
    final ByteArrayDataInput in = new ByteArrayDataInput();
    int pos = -1;
    for (final TokenInfo info : infos) {
        assertThat(ts.incrementToken()).isTrue();
        pos += posIncrAtt.getPositionIncrement();
        int len = -1;
        final BytesRef payload = payloadAtt.getPayload();
        if (info.len != -1) {
            assertThat(payload).isNotNull();
            in.reset(payload.bytes);
            len = in.readVInt();
        } else {
            assertThat(payload).isNull();
        }
        assertThat(new TokenInfo(term.toString(), pos, len)).isEqualTo(info);
    }
    assertThat(ts.incrementToken()).isFalse();
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testEncoder() throws Exception {
  Map<String,String> args = new HashMap<String, String>();
  args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, "float");
  DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
  factory.init(args);
  ResourceLoader loader = new StringMockResourceLoader("solr/collection1");
  factory.inform(loader);

  TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
  DelimitedPayloadTokenFilter tf = factory.create(input);
  tf.reset();
  while (tf.incrementToken()){
    PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
    assertTrue("payAttr is null and it shouldn't be", payAttr != null);
    byte[] payData = payAttr.getPayload().bytes;
    assertTrue("payData is null and it shouldn't be", payData != null);
    assertTrue("payData is null and it shouldn't be", payData != null);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertTrue(payFloat + " does not equal: " + 0.1f, payFloat == 0.1f);
  }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testDelim() throws Exception {
  Map<String,String> args = new HashMap<String, String>();
  args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, FloatEncoder.class.getName());
  args.put(DelimitedPayloadTokenFilterFactory.DELIMITER_ATTR, "*");
  DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
  factory.init(args);
  ResourceLoader loader = new StringMockResourceLoader("solr/collection1");
  factory.inform(loader);

  TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false);
  DelimitedPayloadTokenFilter tf = factory.create(input);
  tf.reset();
  while (tf.incrementToken()){
    PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
    assertTrue("payAttr is null and it shouldn't be", payAttr != null);
    byte[] payData = payAttr.getPayload().bytes;
    assertTrue("payData is null and it shouldn't be", payData != null);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertTrue(payFloat + " does not equal: " + 0.1f, payFloat == 0.1f);
  }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
void assertTermEquals(String expected, TokenStream stream, byte[] expectPay) throws Exception {
  CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
  PayloadAttribute payloadAtt = stream.getAttribute(PayloadAttribute.class);
  stream.reset();
  assertTrue(stream.incrementToken());
  assertEquals(expected, termAtt.toString());
  BytesRef payload = payloadAtt.getPayload();
  if (payload != null) {
    assertTrue(payload.length + " does not equal: " + expectPay.length, payload.length == expectPay.length);
    for (int i = 0; i < expectPay.length; i++) {
      assertTrue(expectPay[i] + " does not equal: " + payload.bytes[i + payload.offset], expectPay[i] == payload.bytes[i + payload.offset]);

    }
  } else {
    assertTrue("expectPay is not null and it should be", expectPay == null);
  }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
    if (input.incrementToken()) {
        CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class);
        final String term = termAtt.toString();
        termAtt.setEmpty();

        PayloadAttribute payloadAtt = this.getAttribute(PayloadAttribute.class);
        final BytesRef payload = payloadAtt.getPayload();
        if(payload == null) {
            return true;
        }

        float payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
        if(payloadValue == 0.0f){
            return true;
        }

        String weight = Float.toString(payloadValue);
        // set weights to zero if in scientific notation
        if(weight.contains("E-")){
            return true;
        }

        String boostedTerm = term + "^" + weight;
        termAtt.append(boostedTerm);
        return true;
    }
    return false;
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
private void assertPayload(int left, int right, int depth, int parent) {
	PayloadAttribute payloadAttribute = tokenizer
			.getAttribute(PayloadAttribute.class);
	BytesRef payload = payloadAttribute.getPayload();
	assertEquals("Incorrect left payload", left,
			payload.bytes[payload.offset + 0]);
	assertEquals("Incorrect right payload", right,
			payload.bytes[payload.offset + 1]);
	assertEquals("Incorrect depth payload", depth,
			payload.bytes[payload.offset + 2]);
	assertEquals("Incorrect parent payload", parent,
			payload.bytes[payload.offset + 3]);
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testVariableTokenPresence() throws IOException {
  String test = "The Quick Red Fox Jumped Over The Lazy Brown Dogs";
  TokenTypeJoinFilter ttjf = new TokenTypeJoinFilter(new Blah2(whitespaceMockTokenizer(test)), new String[] {"raw", "lower", "upper"}, 
      "joined", null, "!", false, false);
  CharTermAttribute termAtt = ttjf.getAttribute(CharTermAttribute.class);
  PayloadAttribute payloadAtt = ttjf.getAttribute(PayloadAttribute.class);
  ttjf.reset();
  int i = -1;
  String[] split = test.split(" ");
  StringBuilder sb = new StringBuilder();
  while (ttjf.incrementToken()) {
    String term = termAtt.toString();
    BytesRef payload = payloadAtt.getPayload();
    switch (++i) {
      case 0:
        assertEquals(split[i], term);
        assertNull(payload);
        break;
      case 1:
        sb.setLength(0);
        sb.append(split[i]).append('!').append(split[i].toUpperCase());
        assertEquals(sb.toString(), term);
        assertNull(payload);
        break;
      case 2:
        sb.setLength(0);
        sb.append(split[i]).append('!').append(split[i].toLowerCase()).append('!').append(split[i].toUpperCase());
        assertEquals(sb.toString(), term);
        assertNull(payload);
        break;
    }
  }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
private void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, byte[] expectPay) throws Exception {
    assertTrue(stream.incrementToken());
    assertEquals(expected, termAtt.toString());
    BytesRef payload = payAtt.getPayload();
    if (payload != null) {
        assertTrue(payload.length == expectPay.length);
        for (int i = 0; i < expectPay.length; i++) {
            assertTrue(expectPay[i] == payload.bytes[payload.offset + i]);
        }
    } else {
        assertTrue(expectPay == null);
    }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public static void displayTokensWithFullDetails(Analyzer analyzer, String text)
    throws IOException {
  
  TokenStream stream = analyzer.tokenStream("contents",
      new StringReader(text));
  
  CharTermAttribute term = stream.addAttribute(CharTermAttribute.class);
  PositionIncrementAttribute posIncr = stream
      .addAttribute(PositionIncrementAttribute.class);
  OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class);
  TypeAttribute type = stream.addAttribute(TypeAttribute.class);
  PayloadAttribute payload = stream.addAttribute(PayloadAttribute.class);
  
  int position = 0;
  while (stream.incrementToken()) {
    
    int increment = posIncr.getPositionIncrement();
    if (increment > 0) {
      position = position + increment;
      System.out.println();
      System.out.print(position + ":");
    }
    
    BytesRef pl = payload.getPayload();
    
    if (pl != null) {
      System.out.print("[" + term.toString() + ":" + offset.startOffset()
          + "->" + offset.endOffset() + ":" + type.type() + ":"
          + new String(pl.bytes) + "] ");
      
    } else {
      System.out.print("[" + term.toString() + ":" + offset.startOffset()
          + "->" + offset.endOffset() + ":" + type.type() + "] ");
      
    }
    
  }
  System.out.println();
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
/**
 * Adds term weights found by tokenizing text from reader into the Map words
 *
 * @param reader a source of text to be tokenized
 * @param termWeightMap a Map of terms and their weights
 * @param fieldName Used by analyzer for any special per-field analysis
 */
private void addTermWeights(Reader reader, Map<String, Flt> termWeightMap, String fieldName)
        throws IOException {
    if (analyzer == null) {
        throw new UnsupportedOperationException("To use RelevancyFeedback without " +
                "term vectors, you must provide an Analyzer");
    }

    TokenStream ts = analyzer.tokenStream(fieldName, reader);
    try {
        int tokenCount = 0;
        // for every token
        CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
        PayloadAttribute payloadAttr = ts.addAttribute(PayloadAttribute.class);

        ts.reset();
        while (ts.incrementToken()) {
            String word = termAtt.toString();
            tokenCount++;
            if (tokenCount > maxNumTokensParsedPerField) {
                break;
            }
            if(word.trim().length() == 0){
                continue;
            }
            if (isNoiseWord(word)) {
                continue;
            }

            BytesRef payload = payloadAttr.getPayload();
            float tokenWeight = 1.0f; // 1.0 or payload if set and a payload field
            if(isPayloadField(fieldName) && payload != null){
                tokenWeight = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
            }
            // increment frequency
            Flt termWeight = termWeightMap.get(word);
            if (termWeight == null) {
                termWeightMap.put(word, new Flt(tokenWeight));
            } else {
                termWeight.x += tokenWeight;
            }
        }
        ts.end();
    } finally {
        IOUtils.closeWhileHandlingException(ts);
    }
}
 

import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; //导入方法依赖的package包/类
public void testOutputComponentTypes() throws IOException {
  String test = "The quick red fox jumped over the lazy brown dogs";

  TokenTypeSplitFilter ttsf = new TokenTypeSplitFilter(new Blah(whitespaceMockTokenizer(test)), Collections.singleton("even"),
      Collections.EMPTY_SET, "even_fork", "even_orig");
  TokenTypeSplitFilter ttsfOdd = new TokenTypeSplitFilter(ttsf, Collections.singleton("odd"),
      Collections.EMPTY_SET, "odd_fork", "odd_orig");
  TokenTypeJoinFilter ttjf = new TokenTypeJoinFilter(ttsfOdd, new String[] {"even_orig", "even_fork"}, "joined", null, "!", true, true);
  int count = 0;
  TypeAttribute typeAtt = ttjf.getAttribute(TypeAttribute.class);
  OffsetAttribute offsetAtt = ttjf.getAttribute(OffsetAttribute.class);
  PositionIncrementAttribute posIncrAtt = ttjf.getAttribute(PositionIncrementAttribute.class);
  CharTermAttribute termAtt = ttjf.getAttribute(CharTermAttribute.class);
  PayloadAttribute payloadAtt = ttjf.getAttribute(PayloadAttribute.class);
  String lastTerm = null;
  int lastStartOffset = -1;
  int lastEndOffset = -1;
  ttjf.reset();
  while (ttjf.incrementToken()) {
    String term = termAtt.toString();
    String type = typeAtt.type();
    int startOffset = offsetAtt.startOffset();
    int endOffset = offsetAtt.endOffset();
    int posIncr = posIncrAtt.getPositionIncrement();
    BytesRef payload = payloadAtt.getPayload();
    switch (count % 5) {
      case 0:
        assertEquals("even_orig", type);
        assertEquals(1, posIncr);
        assertEquals(lastEndOffset + 1, startOffset);
        assertNull(payload);
        break;
      case 1:
        assertEquals("even_fork", type);
        assertEquals(lastTerm, term);
        assertEquals(0, posIncr);
        assertEquals(lastStartOffset, startOffset);
        assertEquals(lastEndOffset, endOffset);
        assertNull(payload);
        break;
      case 2:
        assertEquals("joined", type);
        assertEquals(0, posIncr);
        assertEquals(lastStartOffset, startOffset);
        String[] split = term.split("!");
        assertEquals(split[0], split[1]);
        assertNull(payload);
        break;
      case 3:
        assertEquals("odd_orig", type);
        assertEquals(1, posIncr);
        assertEquals(lastEndOffset + 1, startOffset);
        assertNull(payload);
        break;
      case 4:
        assertEquals("odd_fork", type);
        assertEquals(lastTerm, term);
        assertEquals(0, posIncr);
        assertEquals(lastStartOffset, startOffset);
        assertEquals(lastEndOffset, endOffset);
        assertNull(payload);
        break;
    }
    lastTerm = term;
    lastStartOffset = startOffset;
    lastEndOffset = endOffset;
    count++;
  }
  assertTrue(count + " does not equal: " + 25, count == 25);

}