Java HConstants.OLDEST_TIMESTAMP屬性代碼示例

@Override
public InternalScanner preFlushScannerOpen(
    final ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
  Long newTtl = ttls.get(store.getTableName());
  if (newTtl != null) {
    System.out.println("PreFlush:" + newTtl);
  }
  Integer newVersions = versions.get(store.getTableName());
  ScanInfo oldSI = store.getScanInfo();
  HColumnDescriptor family = store.getFamily();
  ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(),
      family.getName(), family.getMinVersions(),
      newVersions == null ? family.getMaxVersions() : newVersions,
      newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
      oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
  Scan scan = new Scan();
  scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
  return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
      ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
      HConstants.OLDEST_TIMESTAMP);
}
 

/**
 * Extracts some details about the files to compact that are commonly needed by compactors.
 *
 * @param filesToCompact Files.
 * @param allFiles       Whether all files are included for compaction
 * @return The result.
 */
protected FileDetails getFileDetails(Collection<StoreFile> filesToCompact, boolean allFiles)
    throws IOException {
  FileDetails fd = new FileDetails();
  long oldestHFileTimeStampToKeepMVCC =
      System.currentTimeMillis() - (1000L * 60 * 60 * 24 * this.keepSeqIdPeriod);

  for (StoreFile file : filesToCompact) {
    if (allFiles && (file.getModificationTimeStamp() < oldestHFileTimeStampToKeepMVCC)) {
      // when isAllFiles is true, all files are compacted so we can calculate the smallest
      // MVCC value to keep
      if (fd.minSeqIdToKeep < file.getMaxMemstoreTS()) {
        fd.minSeqIdToKeep = file.getMaxMemstoreTS();
      }
    }
    long seqNum = file.getMaxSequenceId();
    fd.maxSeqId = Math.max(fd.maxSeqId, seqNum);
    StoreFile.Reader r = file.getReader();
    if (r == null) {
      LOG.warn("Null reader for " + file.getPath());
      continue;
    }
    // NOTE: use getEntries when compacting instead of getFilterEntries, otherwise under-sized
    // blooms can cause progress to be miscalculated or if the user switches bloom
    // type (e.g. from ROW to ROWCOL)
    long keyCount = r.getEntries();
    fd.maxKeyCount += keyCount;
    // calculate the latest MVCC readpoint in any of the involved store files
    Map<byte[], byte[]> fileInfo = r.loadFileInfo();
    byte tmp[] = null;
    // Get and set the real MVCCReadpoint for bulk loaded files, which is the
    // SeqId number.
    if (r.isBulkLoaded()) {
      fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, r.getSequenceID());
    } else {
      tmp = fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
      if (tmp != null) {
        fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, Bytes.toLong(tmp));
      }
    }
    tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
    if (tmp != null) {
      fd.maxTagsLength = Math.max(fd.maxTagsLength, Bytes.toInt(tmp));
    }
    // If required, calculate the earliest put timestamp of all involved storefiles.
    // This is used to remove family delete marker during compaction.
    long earliestPutTs = 0;
    if (allFiles) {
      tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
      if (tmp == null) {
        // There's a file with no information, must be an old one
        // assume we have very old puts
        fd.earliestPutTs = earliestPutTs = HConstants.OLDEST_TIMESTAMP;
      } else {
        earliestPutTs = Bytes.toLong(tmp);
        fd.earliestPutTs = Math.min(fd.earliestPutTs, earliestPutTs);
      }
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("Compacting " + file + ", keycount=" + keyCount + ", bloomtype=" + r
          .getBloomFilterType().toString() + ", size=" + TraditionalBinaryPrefix
          .long2String(r.length(), "", 1) + ", encoding=" + r.getHFileReader()
          .getDataBlockEncoding() + ", seqNum=" + seqNum + (allFiles ?
          ", earliestPutTs=" + earliestPutTs :
          ""));
    }
  }
  return fd;
}
 

@Override
public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
  Scan scan = new Scan();
  scan.setFilter(new NoDataFilter());
  return new StoreScanner(store, store.getScanInfo(), scan,
      Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
      store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
    long earliestPutTs, InternalScanner s) throws IOException {
  Scan scan = new Scan();
  scan.setFilter(new NoDataFilter());
  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
      ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(),
      HConstants.OLDEST_TIMESTAMP);
}
 

/**
 * Reimplement the default behavior
 */
@Override
public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
  ScanInfo oldSI = store.getScanInfo();
  ScanInfo scanInfo = new ScanInfo(oldSI.getConfiguration(), store.getFamily(), oldSI.getTtl(),
      oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
  Scan scan = new Scan();
  scan.setMaxVersions(oldSI.getMaxVersions());
  return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
      ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
}
 

private void testDropDeletes(
    byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo =
    new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator);
  NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);

  ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
      HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual =
      new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setRow(row, 0, (short)row.length);
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 

@Override
public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
  ScanInfo scanInfo = getScanInfo(store, c.getEnvironment());
  if (scanInfo == null) {
    // take default action
    return null;
  }
  Scan scan = new Scan();
  scan.setMaxVersions(scanInfo.getMaxVersions());
  return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner),
      ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
}
 

public void testDeleteMarkerLongevity() throws Exception {
  try {
    final long now = System.currentTimeMillis();
    EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
      public long currentTime() {
        return now;
      }
    });
    KeyValue[] kvs = new KeyValue[]{
      /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 100, KeyValue.Type.DeleteFamily), // live
      /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 1000, KeyValue.Type.DeleteFamily), // expired
      /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now - 50,
      KeyValue.Type.Put, "v3"), // live
      /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55,
      KeyValue.Type.Delete, "dontcare"), // live
      /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55,
      KeyValue.Type.Put, "deleted-version v2"), // deleted
      /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now - 60,
      KeyValue.Type.Put, "v1"), // live
      /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now - 65,
      KeyValue.Type.Put, "v0"), // max-version reached
      /*7*/ KeyValueTestUtil.create("R1", "cf", "a",
      now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
      /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now - 600,
      KeyValue.Type.DeleteColumn, "dont-care"), //expired
      /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now - 70,
      KeyValue.Type.Put, "v2"), //live
      /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now - 750,
      KeyValue.Type.Put, "v1"), //expired
      /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now - 500,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now - 600,
      KeyValue.Type.Put, "v1"), //expired
      /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now - 1000,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now - 60,
      KeyValue.Type.Put, "expired put"), //live
      /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now - 100,
      KeyValue.Type.Delete, "not-expired delete"), //live
    };
    List<KeyValueScanner> scanners = scanFixture(kvs);
    Scan scan = new Scan();
    scan.setMaxVersions(2);
    ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
      0 /* minVersions */,
      2 /* maxVersions */, 500 /* ttl */,
      KeepDeletedCells.FALSE /* keepDeletedCells */,
      200, /* timeToPurgeDeletes */
      KeyValue.COMPARATOR);
    StoreScanner scanner =
      new StoreScanner(scan, scanInfo,
        ScanType.COMPACT_DROP_DELETES, null, scanners,
        HConstants.OLDEST_TIMESTAMP);
    List<Cell> results = new ArrayList<Cell>();
    results = new ArrayList<Cell>();
    assertEquals(true, scanner.next(results));
    assertEquals(kvs[0], results.get(0));
    assertEquals(kvs[2], results.get(1));
    assertEquals(kvs[3], results.get(2));
    assertEquals(kvs[5], results.get(3));
    assertEquals(kvs[9], results.get(4));
    assertEquals(kvs[14], results.get(5));
    assertEquals(kvs[15], results.get(6));
    assertEquals(7, results.size());
    scanner.close();
  }finally{
  EnvironmentEdgeManagerTestHelper.reset();
  }
}