Java PayloadHelper类代码示例

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
float payloadBoost() throws IOException {
    if (doc != docID()) {
        final int freq = postings.freq();
        payloadBoost = 0;
        for (int i = 0; i < freq; ++i) {
            postings.nextPosition();
            final BytesRef payload = postings.getPayload();
            float boost;
            if (payload == null) {
                boost = 1;
            } else if (payload.length == 1) {
                boost = SmallFloat.byte315ToFloat(payload.bytes[payload.offset]);
            } else if (payload.length == 4) {
                // TODO: for bw compat only, remove this in 6.0
                boost = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
            } else {
                throw new IllegalStateException("Payloads are expected to have a length of 1 or 4 but got: "
                    + payload);
            }
            payloadBoost += boost;
        }
        payloadBoost /= freq;
        doc = docID();
    }
    return payloadBoost;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
private int position(int doc, BytesRef term) {
    int maxPosition = 20;
    try {
        Terms terms = context.reader().getTermVector(doc, weight.field);
        TermsEnum termsEnum = terms.iterator();
        if (!termsEnum.seekExact(term)) {
            Loggers.getLogger(this.getClass()).error("seekExact failed, returning default position = " +
                    maxPosition + " for field = " + weight.field);
            return maxPosition;
        }
        PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL);
        dpEnum.nextDoc();
        dpEnum.nextPosition();
        BytesRef payload = dpEnum.getPayload();
        if (payload == null) {
            Loggers.getLogger(this.getClass()).error("getPayload failed, returning default position = " +
                    maxPosition + " for field = " + weight.field);
            return maxPosition;
        }
        return PayloadHelper.decodeInt(payload.bytes, payload.offset);
    } catch (Exception ex) {
        Loggers.getLogger(this.getClass()).error("Unexpected exception, returning default position = " +
                maxPosition + " for field = " + weight.field, ex);
        return maxPosition;
    }
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public void testEncoder() throws Exception {
  Reader reader = new StringReader("the|0.1 quick|0.1 red|0.1");
  TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
  stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);

  stream.reset();
  while (stream.incrementToken()) {
    PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
    assertNotNull(payAttr);
    byte[] payData = payAttr.getPayload().bytes;
    assertNotNull(payData);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertEquals(0.1f, payFloat, 0.0f);
  }
  stream.end();
  stream.close();
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public void testDelim() throws Exception {
  Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
  TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
  stream = tokenFilterFactory("DelimitedPayload",
      "encoder", "float",
      "delimiter", "*").create(stream);
  stream.reset();
  while (stream.incrementToken()) {
    PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
    assertNotNull(payAttr);
    byte[] payData = payAttr.getPayload().bytes;
    assertNotNull(payData);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertEquals(0.1f, payFloat, 0.0f);
  }
  stream.end();
  stream.close();
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public void testEncoder() throws Exception {
  Map<String,String> args = new HashMap<String, String>();
  args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, "float");
  DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
  factory.init(args);
  ResourceLoader loader = new StringMockResourceLoader("solr/collection1");
  factory.inform(loader);

  TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
  DelimitedPayloadTokenFilter tf = factory.create(input);
  tf.reset();
  while (tf.incrementToken()){
    PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
    assertTrue("payAttr is null and it shouldn't be", payAttr != null);
    byte[] payData = payAttr.getPayload().bytes;
    assertTrue("payData is null and it shouldn't be", payData != null);
    assertTrue("payData is null and it shouldn't be", payData != null);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertTrue(payFloat + " does not equal: " + 0.1f, payFloat == 0.1f);
  }
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public void testDelim() throws Exception {
  Map<String,String> args = new HashMap<String, String>();
  args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, FloatEncoder.class.getName());
  args.put(DelimitedPayloadTokenFilterFactory.DELIMITER_ATTR, "*");
  DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
  factory.init(args);
  ResourceLoader loader = new StringMockResourceLoader("solr/collection1");
  factory.inform(loader);

  TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false);
  DelimitedPayloadTokenFilter tf = factory.create(input);
  tf.reset();
  while (tf.incrementToken()){
    PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
    assertTrue("payAttr is null and it shouldn't be", payAttr != null);
    byte[] payData = payAttr.getPayload().bytes;
    assertTrue("payData is null and it shouldn't be", payData != null);
    float payFloat = PayloadHelper.decodeFloat(payData);
    assertTrue(payFloat + " does not equal: " + 0.1f, payFloat == 0.1f);
  }
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public float payloadAsFloat(float defaultMissing) {
    if (payload != null && payload.length != 0) {
        return PayloadHelper.decodeFloat(payload.bytes, payload.offset);
    } else {
        return defaultMissing;
    }
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
public int payloadAsInt(int defaultMissing) {
    if (payload != null && payload.length != 0) {
        return PayloadHelper.decodeInt(payload.bytes, payload.offset);
    } else {
        return defaultMissing;
    }
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
private String createString(String[] tokens, Map<String, List<BytesRef>> payloads, int encoding, char delimiter) {
    String resultString = "";
    ObjectIntHashMap<String> payloadCounter = new ObjectIntHashMap<>();
    for (String token : tokens) {
        if (!payloadCounter.containsKey(token)) {
            payloadCounter.putIfAbsent(token, 0);
        } else {
            payloadCounter.put(token, payloadCounter.get(token) + 1);
        }
        resultString = resultString + token;
        BytesRef payload = payloads.get(token).get(payloadCounter.get(token));
        if (payload.length > 0) {
            resultString = resultString + delimiter;
            switch (encoding) {
            case 0: {
                resultString = resultString + Float.toString(PayloadHelper.decodeFloat(payload.bytes, payload.offset));
                break;
            }
            case 1: {
                resultString = resultString + Integer.toString(PayloadHelper.decodeInt(payload.bytes, payload.offset));
                break;
            }
            case 2: {
                resultString = resultString + payload.utf8ToString();
                break;
            }
            default: {
                throw new ElasticsearchException("unsupported encoding type");
            }
            }
        }
        resultString = resultString + " ";
    }
    return resultString;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
private Map<String, List<BytesRef>> createPayloads(String[] tokens, int encoding) {
    Map<String, List<BytesRef>> payloads = new HashMap<>();
    for (String token : tokens) {
        if (payloads.get(token) == null) {
            payloads.put(token, new ArrayList<BytesRef>());
        }
        boolean createPayload = randomBoolean();
        if (createPayload) {
            switch (encoding) {
            case 0: {
                float theFloat = randomFloat();
                payloads.get(token).add(new BytesRef(PayloadHelper.encodeFloat(theFloat)));
                break;
            }
            case 1: {
                payloads.get(token).add(new BytesRef(PayloadHelper.encodeInt(randomInt())));
                break;
            }
            case 2: {
                String payload = randomUnicodeOfLengthBetween(50, 100);
                for (int c = 0; c < payload.length(); c++) {
                    if (Character.isWhitespace(payload.charAt(c))) {
                        payload = payload.replace(payload.charAt(c), 'w');
                    }
                }
                payloads.get(token).add(new BytesRef(payload));
                break;
            }
            default: {
                throw new ElasticsearchException("unsupported encoding type");
            }
            }
        } else {
            payloads.get(token).add(new BytesRef());
        }
    }
    return payloads;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
@Override
public final boolean incrementToken() throws IOException {
    if (input.incrementToken()) {
        CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class);
        final String term = termAtt.toString();
        termAtt.setEmpty();

        PayloadAttribute payloadAtt = this.getAttribute(PayloadAttribute.class);
        final BytesRef payload = payloadAtt.getPayload();
        if(payload == null) {
            return true;
        }

        float payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
        if(payloadValue == 0.0f){
            return true;
        }

        String weight = Float.toString(payloadValue);
        // set weights to zero if in scientific notation
        if(weight.contains("E-")){
            return true;
        }

        String boostedTerm = term + "^" + weight;
        termAtt.append(boostedTerm);
        return true;
    }
    return false;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
protected void setAttributes(String token, float payload) {
    CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class);
    termAtt.setEmpty();
    termAtt.append(token);
    termAtt.setLength(token.length());

    PayloadAttribute payloadAtt = this.getAttribute(PayloadAttribute.class);
    byte[] bytes = PayloadHelper.encodeFloat(payload);
    payloadAtt.setPayload(new BytesRef(bytes));
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
@Override
public float scorePayload(int doc, int start, int end, BytesRef payload) {
    if (payload != null) {
        float x = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
        return x;
    }
    return 1.0F;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
@Override
public float scorePayload(int docID, int start, int end, BytesRef payload) {
    float pload = 1.0f;
    if (payload != null) {
        //pload = PayloadHelper.decodeFloat(payload.bytes);
        pload = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
    }
    System.out.println("===>  docid: " + docID + " payload: " + pload);
    return pload;
}
 

import org.apache.lucene.analysis.payloads.PayloadHelper; //导入依赖的package包/类
@Override
public float scorePayload(int docID, int start, int end, BytesRef payload) {

    float pload = 1.0f;
    if (payload != null) {
        pload = PayloadHelper.decodeFloat(payload.bytes);
    }

    logger.info("===>  docid: " + docID + " payload: " + pload);
    return pload;
}