Java CmpUtil.equals方法代码示例

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
/**
 * Returns an iterator pointing at a specific key. Returns null if the key is
 * not found in the index.
 */
@Override
public DiskBTreeIterator getIterator(byte[] key) throws IOException {
  // read from offset to offset in the vocab structure (right?)
  VocabularyReader.IndexBlockInfo slot = vocabulary.get(key);

  if (slot == null) {
    return null;
  }
  DiskBTreeIterator i = new DiskBTreeIterator(this, slot);
  i.find(key);
  if (CmpUtil.equals(key, i.getKey())) {
    return i;
  }
  return null;
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(TextFeature tf) throws IOException {
  debug_total_count++;

  // first feature - record the feature + store the tf in the buffer
  if (currentFeature == null) {
    currentFeature = tf.feature;
    currentBuffer.offerLast(tf);
    // no point emitting here - threshold should be > 1

  } else if (CmpUtil.equals(tf.feature, currentFeature)) {
    currentBuffer.offerLast(tf);
    emitExtents();

  } else {
    notPassing.incrementBy(currentBuffer.size());
    currentBuffer.clear();

    // now prepare for the next feature
    currentFeature = tf.feature;
    currentBuffer.offerLast(tf);
    currentPassesThreshold = false;
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(NumberedExtent ne) throws IOException {
  debug_total_count++;

  // first feature - record the feature + store the tf in the buffer
  if (currentFeature == null) {
    currentFeature = ne.extentName;
    currentBuffer.offerLast(ne);
    // no point emitting here - threshold should be > 1

  } else if (CmpUtil.equals(ne.extentName, currentFeature)) {
    currentBuffer.offerLast(ne);
    emitExtents();

  } else {
    emitExtents();
    discards.incrementBy(currentBuffer.size());
    currentBuffer.clear();

    // now prepare for the next feature
    currentFeature = ne.extentName;
    currentBuffer.offerLast(ne);
    currentPassesThreshold = false;
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Test
public void testCountUnigrams() throws IOException, IncompatibleProcessorException {
  WordCounter counter = new WordCounter(new FakeParameters(Parameters.create()));
  Document document = new Document();
  PostStep post = new PostStep();

  counter.setProcessor(post);

  document.terms = new ArrayList<>();
  document.terms.add("one");
  document.terms.add("two");
  document.terms.add("one");
  counter.process(document);

  assertEquals(2, post.results.size());

  for (int i = 0; i < post.results.size(); ++i) {
    WordCount wc = post.results.get(i);
    if (CmpUtil.equals(wc.word, ByteUtil.fromString("one"))) {
      assertEquals(2, wc.collectionFrequency);
    } else if (CmpUtil.equals(wc.word, ByteUtil.fromString("one"))) {
      assertEquals(1, wc.collectionFrequency);
    }
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(NumberWordCount current) throws IOException {
  if (last == null) {
    last = current;

  } else if (CmpUtil.equals(last.word, current.word) && last.document == current.document) {
    last.count += current.count;

  } else {
    processor.process(last);
    last = current;
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(NumberWordCount nwc) throws IOException {
  debug_total_count++;

  // first feature - record the feature + store the tf in the buffer
  if (currentFeature == null) {
    currentFeature = nwc.word;
    currentBuffer.offerLast(nwc);
    // no point emitting here - threshold should be > 1

  } else if (CmpUtil.equals(nwc.word, currentFeature)) {
    currentBuffer.offerLast(nwc);
    emitExtents();

  } else {
    emitExtents();
    if (discards != null) {
      for(NumberWordCount c : currentBuffer) {
        discards.incrementBy(c.count);
      }
    }
    currentBuffer.clear();

    // now prepare for the next feature
    currentFeature = nwc.word;
    currentBuffer.offerLast(nwc);
    currentPassesThreshold = false;
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
public boolean skipTo(byte[] key) throws IOException {
  iterator.skipTo(key);
  if (CmpUtil.equals(key, iterator.getKey())) {
    reset();
    return true;
  }
  return false;
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(Document document) throws IOException {
  List<String> tokens = document.terms;
  ArrayList<WordCount> wordCounts = new ArrayList();

  for (String t : tokens) {
    if (t != null) {
      if ((filterWords == null)
              || (!filterWords.contains(t))) {
        wordCounts.add(new WordCount(ByteUtil.fromString(t), 1, 1, 1));
      }
    }
  }

  Collections.sort(wordCounts, new WordCount.WordOrder().lessThan());

  WordCount last = null;

  for (WordCount wc : wordCounts) {
    if (last == null) {
      last = wc;
    } else if (CmpUtil.equals(wc.word, last.word)) {
      last.collectionFrequency += wc.collectionFrequency;
      last.maxDocumentFrequency += wc.maxDocumentFrequency;
    } else {
      processor.process(last);
      last = wc;
    }
  }

  if (last != null) {
    processor.process(last);
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(KeyValuePair kvp) throws IOException {
  if (!namesIterator.isDone()) {
    if (namesIterator.skipToKey(kvp.key)) {
      if (CmpUtil.equals(namesIterator.getKey(), kvp.key)) {
        numbered.increment();
        processor.process(new NumberKeyValue(namesIterator.getCurrentIdentifier(), kvp.key, kvp.value));
      }
    }
  }
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(KeyValuePair kvp) throws IOException {
  if (previous == null){
    processor.process(kvp);
  } else if( CmpUtil.equals(previous.key, kvp.key)  && CmpUtil.equals(previous.value, kvp.value)){
    // identical conflations - already processed previous - so do nothing
  } else {
    // otherwise different conflations - process kvp
    processor.process(kvp);
  }
  // update previous
  previous = kvp;
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(KeyValuePair object) throws IOException {
  // if we have some things to reduce, and this key is new
  if (!current.isEmpty() && !CmpUtil.equals(current.get(current.size() - 1).key, object.key)) {
    flush();
  }
  current.add(object);
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public boolean skipToKey(byte[] key) throws IOException {
  keyIterator = corpusData.tailMap(key).keySet().iterator();
  nextKey();
  return CmpUtil.equals(key, currKey);
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public boolean findKey(byte[] key) throws IOException {
  keyIterator = corpusData.tailMap(key).keySet().iterator();
  nextKey();
  return CmpUtil.equals(key, currKey);
}
 

import org.lemurproject.galago.utility.CmpUtil; //导入方法依赖的package包/类
@Override
public void process(DocumentSplit split) throws IOException {
  long limit = Long.MAX_VALUE;
  if (split.startKey.length > 0) {
    if (CmpUtil.equals(subCollCheck, split.startKey)) {
      limit = VByte.uncompressLong(split.endKey, 0);
    }
  }

  if (this.documentStreamParser.containsKey(split.fileType)) {
    try {
      Class c = documentStreamParser.get(split.fileType);
      Constructor cstr = c.getConstructor(DocumentSplit.class, Parameters.class);
      DocumentStreamParser parser = (DocumentStreamParser) cstr.newInstance(split, parameters);

      Document document;
      long count = 0;
      while ((document = parser.nextDocument()) != null) {
        document.fileId = split.fileId;
        document.totalFileCount = split.totalFileCount;

        documentCounter.increment();

        count++;

        // Enforces limitations imposed by the endKey subcollection specifier.
        // See DocumentSource for details.
        if (count >= limit) {
          break;
        }
      }

      if (parser != null) {
        parser.close();
      }

      KeyValuePair kvp = new KeyValuePair(ByteUtil.fromString(split.fileName), Utility.fromLong(count));
      processor.process(kvp);

    } catch (Exception ex) {
      logger.log(Level.INFO, "Failed to parse document split - {0} as {1}\n", new Object[]{split.toString(), split.fileType});
      logger.log(Level.SEVERE, ex.toString());
    }
  } else {
    logger.log(Level.INFO, "Ignoring {0} - could not find a parser for file-type:{1}\n", new Object[]{split.toString(), split.fileType});
  }
}