本文整理汇总了Java中org.apache.hadoop.hbase.wal.WAL.append方法的典型用法代码示例。如果您正苦于以下问题:Java WAL.append方法的具体用法?Java WAL.append怎么用?Java WAL.append使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.hbase.wal.WAL
的用法示例。
在下文中一共展示了WAL.append方法的11个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: addWALEdits
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
String familyStr = Bytes.toString(family);
long txid = -1;
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
// uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
// about legacy coprocessors
txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
ee.currentTime(), mvcc), edit, true);
}
if (-1 != txid) {
wal.sync(txid);
}
}
示例2: addWALEdits
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc)
throws IOException {
String familyStr = Bytes.toString(family);
for (int j = 0; j < count; j++) {
byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
WALEdit edit = new WALEdit();
edit.add(new KeyValue(rowName, family, qualifierBytes,
ee.currentTime(), columnBytes));
wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,999, mvcc),
edit, true);
}
wal.sync();
}
示例3: writeMarker
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private static long writeMarker(final WAL wal, final HTableDescriptor htd, final HRegionInfo hri,
final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync)
throws IOException {
// TODO: Pass in current time to use?
WALKey key =
new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc);
// Add it to the log but the false specifies that we don't need to add it to the memstore
long trx = MultiVersionConcurrencyControl.NONE;
try {
trx = wal.append(htd, hri, key, edit, false);
if (sync) wal.sync(trx);
} finally {
// If you get hung here, is it a real WAL or a mocked WAL? If the latter, you need to
// trip the latch that is inside in getWriteEntry up in your mock. See down in the append
// called from onEvent in FSHLog.
MultiVersionConcurrencyControl.WriteEntry we = key.getWriteEntry();
if (mvcc != null && we != null) mvcc.complete(we);
}
return trx;
}
示例4: appendEmptyEdit
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
* Append a faked WALEdit in order to get a long sequence number and wal syncer will just ignore
* the WALEdit append later.
*
* @param wal
* @return Return the key used appending with no sync and no append.
* @throws IOException
*/
private WALKey appendEmptyEdit(final WAL wal) throws IOException {
// we use HLogKey here instead of WALKey directly to support legacy
// coprocessors.
@SuppressWarnings("deprecation") WALKey key =
new HLogKey(getRegionInfo().getEncodedNameAsBytes(), getRegionInfo().getTable(),
WALKey.NO_SEQUENCE_ID, 0, null, HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());
// Call append but with an empty WALEdit. The returned sequence id will not
// be associated
// with any edit and we can be sure it went in after all outstanding
// appends.
try {
wal.append(getTableDesc(), getRegionInfo(), key, WALEdit.EMPTY_WALEDIT, false);
} catch (Throwable t) {
// If exception, our mvcc won't get cleaned up by client, so do it here.
getMVCC().complete(key.getWriteEntry());
}
return key;
}
示例5: addEdits
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
protected void addEdits(WAL log,
HRegionInfo hri,
HTableDescriptor htd,
int times,
MultiVersionConcurrencyControl mvcc)
throws IOException {
final byte[] row = Bytes.toBytes("row");
for (int i = 0; i < times; i++) {
long timestamp = System.currentTimeMillis();
WALEdit cols = new WALEdit();
cols.add(new KeyValue(row, row, row, timestamp, row));
WALKey key = new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(),
WALKey.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
HConstants.NO_NONCE, mvcc);
log.append(htd, hri, key, cols, true);
}
log.sync();
}
示例6: testEmptyWALEditAreNotSeen
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
* Coprocessors shouldn't get notice of empty waledits.
*/
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
WAL log = wals.getWAL(UNSPECIFIED_REGION);
try {
SampleRegionWALObserver cp = getCoprocessor(log, SampleRegionWALObserver.class);
cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);
assertFalse(cp.isPreWALWriteCalled());
assertFalse(cp.isPostWALWriteCalled());
final long now = EnvironmentEdgeManager.currentTime();
long txid = log.append(htd, hri,
new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc),
new WALEdit(), true);
log.sync(txid);
assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
} finally {
log.close();
}
}
示例7: testLogRollAfterSplitStart
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
* Tests the case where a RegionServer enters a GC pause,
* comes back online after the master declared it dead and started to split.
* Want log rolling after a master split to fail. See HBASE-2312.
*/
@Test (timeout=300000)
public void testLogRollAfterSplitStart() throws IOException {
LOG.info("Verify wal roll after split starts will fail.");
String logName = "testLogRollAfterSplitStart";
Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
final WALFactory wals = new WALFactory(conf, null, logName);
try {
// put some entries in an WAL
TableName tableName =
TableName.valueOf(this.getClass().getName());
HRegionInfo regioninfo = new HRegionInfo(tableName,
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
final int total = 20;
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor("column"));
log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
System.currentTimeMillis(), mvcc), kvs, true);
}
// Send the data to HDFS datanodes and close the HDFS writer
log.sync();
((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);
/* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog()
* handles RS shutdowns (as observed by the splitting process)
*/
// rename the directory so a rogue RS doesn't create more WALs
Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
if (!fs.rename(thisTestsDir, rsSplitDir)) {
throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
}
LOG.debug("Renamed region directory: " + rsSplitDir);
LOG.debug("Processing the old log files.");
WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);
LOG.debug("Trying to roll the WAL.");
try {
log.rollWriter();
Assert.fail("rollWriter() did not throw any exception.");
} catch (IOException ioe) {
if (ioe.getCause() instanceof FileNotFoundException) {
LOG.info("Got the expected exception: ", ioe.getCause());
} else {
Assert.fail("Unexpected exception: " + ioe);
}
}
} finally {
wals.close();
if (fs.exists(thisTestsDir)) {
fs.delete(thisTestsDir, true);
}
}
}
示例8: testActionListener
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
* Add a bunch of dummy data and roll the logs every two insert. We
* should end up with 10 rolled files (plus the roll called in
* the constructor). Also test adding a listener while it's running.
*/
@Test
public void testActionListener() throws Exception {
DummyWALActionsListener observer = new DummyWALActionsListener();
List<WALActionsListener> list = new ArrayList<WALActionsListener>();
list.add(observer);
final WALFactory wals = new WALFactory(conf, list, "testActionListener");
DummyWALActionsListener laterobserver = new DummyWALActionsListener();
HRegionInfo hri = new HRegionInfo(TableName.valueOf(SOME_BYTES),
SOME_BYTES, SOME_BYTES, false);
final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes());
for (int i = 0; i < 20; i++) {
byte[] b = Bytes.toBytes(i+"");
KeyValue kv = new KeyValue(b,b,b);
WALEdit edit = new WALEdit();
edit.add(kv);
HTableDescriptor htd = new HTableDescriptor();
htd.addFamily(new HColumnDescriptor(b));
final long txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(),
TableName.valueOf(b), 0), edit, true);
wal.sync(txid);
if (i == 10) {
wal.registerWALActionsListener(laterobserver);
}
if (i % 2 == 0) {
wal.rollWriter();
}
}
wal.close();
assertEquals(11, observer.preLogRollCounter);
assertEquals(11, observer.postLogRollCounter);
assertEquals(5, laterobserver.preLogRollCounter);
assertEquals(5, laterobserver.postLogRollCounter);
assertEquals(1, observer.closedCount);
}
示例9: testLogRoll
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void testLogRoll() throws Exception {
long baseline = 1000;
long time = baseline;
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
KeyValue kv = new KeyValue(r1, f1, r1);
WALEdit edit = new WALEdit();
edit.add(kv);
List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
listeners.add(replication);
final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners,
URLEncoder.encode("regionserver:60020", "UTF8"));
final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes());
manager.init();
HTableDescriptor htd = new HTableDescriptor();
htd.addFamily(new HColumnDescriptor(f1));
// Testing normal log rolling every 20
for(long i = 1; i < 101; i++) {
if(i > 1 && i % 20 == 0) {
wal.rollWriter();
}
LOG.info(i);
final long txid = wal.append(htd,
hri,
new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
edit,
true);
wal.sync(txid);
}
// Simulate a rapid insert that's followed
// by a report that's still not totally complete (missing last one)
LOG.info(baseline + " and " + time);
baseline += 101;
time = baseline;
LOG.info(baseline + " and " + time);
for (int i = 0; i < 3; i++) {
wal.append(htd, hri,
new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
edit,
true);
}
wal.sync();
int logNumber = 0;
for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
logNumber += entry.getValue().size();
}
assertEquals(6, logNumber);
wal.rollWriter();
manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
"1", 0, false, false);
wal.append(htd, hri,
new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
edit,
true);
wal.sync();
assertEquals(1, manager.getWALs().size());
// TODO Need a case with only 2 WALs and we only want to delete the first one
}
示例10: testNonLegacyWALKeysDoNotExplode
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void testNonLegacyWALKeysDoNotExplode() throws Exception {
TableName tableName = TableName.valueOf(TEST_TABLE);
final HTableDescriptor htd = createBasic3FamilyHTD(Bytes
.toString(TEST_TABLE));
final HRegionInfo hri = new HRegionInfo(tableName, null, null);
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
fs.mkdirs(new Path(FSUtils.getTableDir(hbaseRootDir, tableName), hri.getEncodedName()));
final Configuration newConf = HBaseConfiguration.create(this.conf);
final WAL wal = wals.getWAL(UNSPECIFIED_REGION);
final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class);
newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
final SampleRegionWALObserver oldApi = getCoprocessor(wal,
SampleRegionWALObserver.Legacy.class);
oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
LOG.debug("ensuring wal entries haven't happened before we start");
assertFalse(newApi.isPreWALWriteCalled());
assertFalse(newApi.isPostWALWriteCalled());
assertFalse(newApi.isPreWALWriteDeprecatedCalled());
assertFalse(newApi.isPostWALWriteDeprecatedCalled());
assertFalse(oldApi.isPreWALWriteCalled());
assertFalse(oldApi.isPostWALWriteCalled());
assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
assertFalse(oldApi.isPostWALWriteDeprecatedCalled());
LOG.debug("writing to WAL with non-legacy keys.");
final int countPerFamily = 5;
for (HColumnDescriptor hcd : htd.getFamilies()) {
addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
}
LOG.debug("Verify that only the non-legacy CP saw edits.");
assertTrue(newApi.isPreWALWriteCalled());
assertTrue(newApi.isPostWALWriteCalled());
assertFalse(newApi.isPreWALWriteDeprecatedCalled());
assertFalse(newApi.isPostWALWriteDeprecatedCalled());
// wish we could test that the log message happened :/
assertFalse(oldApi.isPreWALWriteCalled());
assertFalse(oldApi.isPostWALWriteCalled());
assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
assertFalse(oldApi.isPostWALWriteDeprecatedCalled());
LOG.debug("reseting cp state.");
newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
LOG.debug("write a log edit that supports legacy cps.");
final long now = EnvironmentEdgeManager.currentTime();
final WALKey legacyKey = new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now);
final WALEdit edit = new WALEdit();
final byte[] nonce = Bytes.toBytes("1772");
edit.add(new KeyValue(TEST_ROW, TEST_FAMILY[0], nonce, now, nonce));
final long txid = wal.append(htd, hri, legacyKey, edit, true);
wal.sync(txid);
LOG.debug("Make sure legacy cps can see supported edits after having been skipped.");
assertTrue("non-legacy WALObserver didn't see pre-write.", newApi.isPreWALWriteCalled());
assertTrue("non-legacy WALObserver didn't see post-write.", newApi.isPostWALWriteCalled());
assertFalse("non-legacy WALObserver shouldn't have seen legacy pre-write.",
newApi.isPreWALWriteDeprecatedCalled());
assertFalse("non-legacy WALObserver shouldn't have seen legacy post-write.",
newApi.isPostWALWriteDeprecatedCalled());
assertTrue("legacy WALObserver didn't see pre-write.", oldApi.isPreWALWriteCalled());
assertTrue("legacy WALObserver didn't see post-write.", oldApi.isPostWALWriteCalled());
assertTrue("legacy WALObserver didn't see legacy pre-write.",
oldApi.isPreWALWriteDeprecatedCalled());
assertTrue("legacy WALObserver didn't see legacy post-write.",
oldApi.isPostWALWriteDeprecatedCalled());
}
示例11: testWALCoprocessorReplay
import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
* Test WAL replay behavior with WALObserver.
*/
@Test
public void testWALCoprocessorReplay() throws Exception {
// WAL replay is handled at HRegion::replayRecoveredEdits(), which is
// ultimately called by HRegion::initialize()
TableName tableName = TableName.valueOf("testWALCoprocessorReplay");
final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
// final HRegionInfo hri =
// createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
// final HRegionInfo hri1 =
// createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
final HRegionInfo hri = new HRegionInfo(tableName, null, null);
final Path basedir =
FSUtils.getTableDir(this.hbaseRootDir, tableName);
deleteDir(basedir);
fs.mkdirs(new Path(basedir, hri.getEncodedName()));
final Configuration newConf = HBaseConfiguration.create(this.conf);
// WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
WAL wal = wals.getWAL(UNSPECIFIED_REGION);
// Put p = creatPutWith2Families(TEST_ROW);
WALEdit edit = new WALEdit();
long now = EnvironmentEdgeManager.currentTime();
// addFamilyMapToWALEdit(p.getFamilyMap(), edit);
final int countPerFamily = 1000;
// for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
for (HColumnDescriptor hcd : htd.getFamilies()) {
addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
}
wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc), edit, true);
// sync to fs.
wal.sync();
User user = HBaseTestingUtility.getDifferentUser(newConf,
".replay.wal.secondtime");
user.runAs(new PrivilegedExceptionAction() {
public Object run() throws Exception {
Path p = runWALSplit(newConf);
LOG.info("WALSplit path == " + p);
FileSystem newFS = FileSystem.get(newConf);
// Make a new wal for new region open.
final WALFactory wals2 = new WALFactory(conf, null, currentTest.getMethodName()+"2");
WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION);;
HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir,
hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
long seqid2 = region.getOpenSeqNum();
SampleRegionWALObserver cp2 =
(SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor(
SampleRegionWALObserver.class.getName());
// TODO: asserting here is problematic.
assertNotNull(cp2);
assertTrue(cp2.isPreWALRestoreCalled());
assertTrue(cp2.isPostWALRestoreCalled());
assertFalse(cp2.isPreWALRestoreDeprecatedCalled());
assertFalse(cp2.isPostWALRestoreDeprecatedCalled());
region.close();
wals2.close();
return null;
}
});
}