Java BatchWriter类代码示例

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
private void processSinglePathWithByteBuffer(BatchWriter writer, FileMapping mapping, Path p, CsvParser parser) throws IOException, MutationsRejectedException {
  final FileSystem fs = p.getFileSystem(conf);
  FSDataInputStream dis = fs.open(p, INPUT_BUFFER_SIZE);
  InputStreamReader reader = new InputStreamReader(dis, UTF_8);
  try {
    parser.beginParsing(reader);
    String[] line = null;
    while ((line = parser.parseNext()) != null) {
      writer.addMutation(parseLine(mapping, line));
    }
  } finally {
    if (null != reader) {
      reader.close();
    }
  }
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
/**
 * Write random entries.
 * <p>
 * Closes the writer after entries are written.
 *
 * @param writer
 *          Writer to write entries to.
 */
void writeRandomEntries(BatchWriter writer) throws MutationsRejectedException {
  for (int i = 0; i < rowCount; i++) {
    byte[] row = getRandomBytes(keyFieldSize, true);

    for (int j = 0; j < columnCount; j++) {
      byte[] colF = getRandomBytes(keyFieldSize, true);
      byte[] colQ = getRandomBytes(keyFieldSize, true);
      byte[] value = getRandomBytes(valueFieldSize, false);

      Mutation mutation = new Mutation(row);
      mutation.put(colF, colQ, VISIBILITY, value);
      writer.addMutation(mutation);
    }
  }

  writer.close();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
/**
 * Writes the given data to Accumulo. The full combinatorial of values is written.
 *
 * @param rows
 *          Rows to write.
 * @param colFs
 *          Column families to write.
 * @param colQs
 *          Column qualifiers to write.
 * @param colVs
 *          Column visibilities to write.
 * @param values
 *          Values to write.
 */
private static void writeData(BatchWriter writer, Iterable<String> rows, Iterable<String> colFs, Iterable<String> colQs, Iterable<String> colVs,
    Iterable<String> values) throws MutationsRejectedException {
  List<Mutation> mutations = new ArrayList<>();

  for (String row : rows) {
    Mutation mutation = new Mutation(row);
    mutations.add(mutation);

    for (String colF : colFs) {
      for (String colQ : colQs) {
        for (String colV : colVs) {
          for (String value : values) {
            mutation.put(colF, colQ, new ColumnVisibility(colV), value);
          }
        }
      }
    }
  }

  writer.addMutations(mutations);
  writer.flush();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Test
public void addMutationsTest() throws Exception {
  when(mockConnector.createBatchWriter(TEST_TABLE, null)).thenReturn(mockWriter);

  BatchWriter writer = new SignedBatchWriter(mockConnector, TEST_TABLE, null, getConfig("config1.ini"), aliceKeyContainers.get(ValueSigner.RSA_PSS));

  List<Mutation> mutations = new ArrayList<>();
  Mutation mutation = new Mutation("row");
  mutation.put("colF", "colQ", "val");

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(1)).addMutation(any()); // 1 time
  verify(mockSignatureWriter, never()).addMutation(any());

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(3)).addMutation(any()); // 1 + 2 times
  verify(mockSignatureWriter, never()).addMutation(any());

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(6)).addMutation(any()); // 1 + 2 + 3 times
  verify(mockSignatureWriter, never()).addMutation(any());
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Test
public void addMutationsExternalTest() throws Exception {
  when(mockConnector.createBatchWriter(TEST_TABLE, null)).thenReturn(mockWriter);
  when(mockConnector.createBatchWriter(SIG_TABLE, null)).thenReturn(mockSignatureWriter);

  BatchWriter writer = new SignedBatchWriter(mockConnector, TEST_TABLE, null, getConfig("config3.ini"), aliceKeyContainers.get(ValueSigner.ECDSA));

  List<Mutation> mutations = new ArrayList<>();
  Mutation mutation = new Mutation("row");
  mutation.put("colF", "colQ", "val");

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(1)).addMutation(any()); // 1 time
  verify(mockSignatureWriter, times(1)).addMutation(any()); // 1 time

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(3)).addMutation(any()); // 1 + 2 times
  verify(mockSignatureWriter, times(3)).addMutation(any()); // 1 + 2 times

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(6)).addMutation(any()); // 1 + 2 + 3 times
  verify(mockSignatureWriter, times(6)).addMutation(any()); // 1 + 2 + 3 times
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Test
public void addMutationsTest() throws Exception {
  when(mockConnector.createBatchWriter(TEST_TABLE, null)).thenReturn(mockWriter);
  BatchWriter writer = new EncryptedBatchWriter(mockConnector, TEST_TABLE, null, getConfig("encrypt-value.ini"), KEYS);

  List<Mutation> mutations = new ArrayList<>();
  Mutation mutation = new Mutation("row");
  mutation.put("colF", "colQ", "val");

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(1)).addMutation(any()); // 1 time

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(3)).addMutation(any()); // 1 + 2 times

  mutations.add(mutation);
  writer.addMutations(mutations);
  verify(mockWriter, times(6)).addMutation(any()); // 1 + 2 + 3 times
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);

  Connector conn = opts.getConnector();
  if (!conn.tableOperations().exists(opts.getTableName())) {
    conn.tableOperations().create(opts.getTableName());
    conn.tableOperations().attachIterator(opts.getTableName(), new IteratorSetting(1, ChunkCombiner.class));
  }
  BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
  FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
  for (String filename : opts.files) {
    fdi.insertFileData(filename, bw);
  }
  bw.close();
  //TODO
  //opts.stopTracing();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, MutationsRejectedException, TableExistsException,
    TableNotFoundException {
  ClientOnRequiredTable opts = new ClientOnRequiredTable();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(InsertWithBatchWriter.class.getName(), args, bwOpts);

  Connector connector = opts.getConnector();
  MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());

  if (!connector.tableOperations().exists(opts.getTableName()))
    connector.tableOperations().create(opts.getTableName());
  BatchWriter bw = mtbw.getBatchWriter(opts.getTableName());

  Text colf = new Text("colfam");
  System.out.println("writing ...");
  for (int i = 0; i < 10000; i++) {
    Mutation m = new Mutation(new Text(String.format("row_%d", i)));
    for (int j = 0; j < 5; j++) {
      m.put(colf, new Text(String.format("colqual_%d", j)), new Value((String.format("value_%d_%d", i, j)).getBytes()));
    }
    bw.addMutation(m);
    if (i % 100 == 0)
      System.out.println(i);
  }
  mtbw.close();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
private void createEntries(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {

    // Trace the write operation. Note, unless you flush the BatchWriter, you will not capture
    // the write operation as it is occurs asynchronously. You can optionally create additional Spans
    // within a given Trace as seen below around the flush
    TraceScope scope = Trace.startSpan("Client Write", Sampler.ALWAYS);

    System.out.println("TraceID: " + Long.toHexString(scope.getSpan().getTraceId()));
    BatchWriter batchWriter = opts.getConnector().createBatchWriter(opts.getTableName(), new BatchWriterConfig());

    Mutation m = new Mutation("row");
    m.put("cf", "cq", "value");

    batchWriter.addMutation(m);
    // You can add timeline annotations to Spans which will be able to be viewed in the Monitor
    scope.getSpan().addTimelineAnnotation("Initiating Flush");
    batchWriter.flush();

    batchWriter.close();
    scope.close();
  }
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
/**
 * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number.
 * The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size.
 */
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
  Opts opts = new Opts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(SequentialBatchWriter.class.getName(), args, bwOpts);
  Connector connector = opts.getConnector();
  BatchWriter bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());

  long end = opts.start + opts.num;

  for (long i = opts.start; i < end; i++) {
    Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
    bw.addMutation(m);
  }

  bw.close();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
public static void index(int numPartitions, File src, String splitRegex, BatchWriter bw) throws Exception {
  if (src.isDirectory()) {
    File[] files = src.listFiles();
    if (files != null) {
      for (File child : files) {
        index(numPartitions, child, splitRegex, bw);
      }
    }
  } else {
    FileReader fr = new FileReader(src);

    StringBuilder sb = new StringBuilder();

    char data[] = new char[4096];
    int len;
    while ((len = fr.read(data)) != -1) {
      sb.append(data, 0, len);
    }

    fr.close();

    index(numPartitions, new Text(src.getAbsolutePath()), sb.toString(), splitRegex, bw);
  }

}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);

  Connector conn = opts.getConnector();

  Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);
  BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());

  for (Entry<Key,Value> entry : scanner) {
    Key key = entry.getKey();
    Mutation m = new Mutation(key.getColumnQualifier());
    m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
    bw.addMutation(m);
  }

  bw.close();

}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Test
public void test() throws Exception {
  conn.tableOperations().create(tableName);
  BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(0, CIFTester.main(tableName, CIFTester.TestMapper.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Test
public void testErrorOnNextWithoutClose() throws Exception {
  conn.tableOperations().create(tableName);
  BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(1, CIFTester.main(tableName, CIFTester.TestNoClose.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
  // this should actually exist, in addition to the dummy entry
  assertEquals(2, assertionErrors.get(tableName + "_map_ioexception").size());
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入依赖的package包/类
@Before
public void setupInstance() throws Exception {
  tableName = getUniqueNames(1)[0];
  conn = getConnector();
  conn.tableOperations().create(tableName);
  BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
  ColumnVisibility cv = new ColumnVisibility();
  // / has 1 dir
  // /local has 2 dirs 1 file
  // /local/user1 has 2 files
  bw.addMutation(Ingest.buildMutation(cv, "/local", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user1", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user2", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 23456, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user1/file1", false, false, false, 2024, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user1/file2", false, false, false, 1028, 23456, null));
  bw.close();
}