Java BatchWriter.flush方法代码示例

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
/**
 * Writes the given data to Accumulo. The full combinatorial of values is written.
 *
 * @param rows
 *          Rows to write.
 * @param colFs
 *          Column families to write.
 * @param colQs
 *          Column qualifiers to write.
 * @param colVs
 *          Column visibilities to write.
 * @param values
 *          Values to write.
 */
private static void writeData(BatchWriter writer, Iterable<String> rows, Iterable<String> colFs, Iterable<String> colQs, Iterable<String> colVs,
    Iterable<String> values) throws MutationsRejectedException {
  List<Mutation> mutations = new ArrayList<>();

  for (String row : rows) {
    Mutation mutation = new Mutation(row);
    mutations.add(mutation);

    for (String colF : colFs) {
      for (String colQ : colQs) {
        for (String colV : colVs) {
          for (String value : values) {
            mutation.put(colF, colQ, new ColumnVisibility(colV), value);
          }
        }
      }
    }
  }

  writer.addMutations(mutations);
  writer.flush();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
private void createEntries(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {

    // Trace the write operation. Note, unless you flush the BatchWriter, you will not capture
    // the write operation as it is occurs asynchronously. You can optionally create additional Spans
    // within a given Trace as seen below around the flush
    TraceScope scope = Trace.startSpan("Client Write", Sampler.ALWAYS);

    System.out.println("TraceID: " + Long.toHexString(scope.getSpan().getTraceId()));
    BatchWriter batchWriter = opts.getConnector().createBatchWriter(opts.getTableName(), new BatchWriterConfig());

    Mutation m = new Mutation("row");
    m.put("cf", "cq", "value");

    batchWriter.addMutation(m);
    // You can add timeline annotations to Spans which will be able to be viewed in the Monitor
    scope.getSpan().addTimelineAnnotation("Initiating Flush");
    batchWriter.flush();

    batchWriter.close();
    scope.close();
  }
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public void testSimpleOutput() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("row");
    row.put("cf", "cq", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a|z&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(1, count);
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public void testColumns() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("a");
    row.put("cf1", "cq", new Value(new byte[0]));
    row.put("cf2", "cq", new Value(new byte[0]));
    row.put("cf3", "cq1", new Value(new byte[0]));
    row.put("cf3", "cq2", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a|c&columns=cf1,cf3|cq1&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(2, count);
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public void testWholeRowRange() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("a");
    row.put("cf1", "cq", new Value(new byte[0]));
    row.put("cf2", "cq", new Value(new byte[0]));
    row.put("cf3", "cq1", new Value(new byte[0]));
    row.put("cf3", "cq2", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(4, count);
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public static void setState(Store id, State state) throws TableNotFoundException, MutationsRejectedException {
  checkNotNull(id);
  checkNotNull(state);
  
  BatchWriter bw = null;
  try {
    bw = id.connector().createBatchWriter(id.metadataTable(), id.writerConfig());
    Mutation m = new Mutation(id.uuid());
    m.put(STATE_COLFAM, EMPTY_TEXT, new Value(state.toString().getBytes()));
    
    bw.addMutation(m);
    bw.flush();
  } finally {
    if (null != bw) {
      bw.close();
    }
  }
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public void run() throws Exception {

    entriesScanned = 0;
    inserts = 0;

    Scanner scanner = conn.createScanner(tableName, auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    BatchWriter bw = conn.createBatchWriter(tableName, bwOpts.getBatchWriterConfig());

    long t1 = System.currentTimeMillis();

    int depth = findMaxDepth(scanner);

    long t2 = System.currentTimeMillis();

    for (int d = depth; d > 0; d--) {
      calculateCounts(scanner, d, bw);
      // must flush so next depth can read what prev depth wrote
      bw.flush();
    }

    bw.close();

    long t3 = System.currentTimeMillis();

    System.out.printf("Max depth              : %d%n", depth);
    System.out.printf("Time to find max depth : %,d ms%n", (t2 - t1));
    System.out.printf("Time to compute counts : %,d ms%n", (t3 - t2));
    System.out.printf("Entries scanned        : %,d %n", entriesScanned);
    System.out.printf("Counts inserted        : %,d %n", inserts);
  }
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
  try {
    for (BatchWriter w : writerMap.values()) {
      w.flush();
      w.close();
    }
  } catch (MutationsRejectedException e) {
    throw new IOException("Error closing Batch Writer", e);
  }
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public static void writeMutations(final Connector connector, final String tableName, final Collection<Mutation> mutations) throws TableNotFoundException, MutationsRejectedException {
    final BatchWriter bw = connector.createBatchWriter(tableName, 10000l, 10000l, 4);
    for(final Mutation mutation : mutations) {
        bw.addMutation(mutation);
    }
    bw.flush();
    bw.close();
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
private void persistTestMutations(int numRows, int entriesPerRow) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++)
                m.put(new Text(Integer.toString(i)), new Text(""), new Value("".getBytes()));

            writer.addMutation(m);
        }
        writer.flush();
    }
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
private void persistTestMutations(int numRows, int entriesPerRow) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++)
                m.put(new Text(Integer.toString(j)), new Text(String.valueOf(i)), new Value("".getBytes()));

            writer.addMutation(m);
        }
        writer.flush();
    }
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
private void persistTestMutations(int numRows) throws TableNotFoundException, MutationsRejectedException {

    BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

    for (int j = 0; j < numRows; j++) {
      Mutation m = new Mutation(Integer.toString(j) + "|suffix");
      m.put(new Text(), new Text(), new Value("".getBytes()));

      for(int i = 0; i < numRows; i++)
        m.put(new Text("" +i), new Text(), new Value("".getBytes()));
      writer.addMutation(m);
    }
    writer.flush();
  }
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public static void flush(BatchWriter writer) {
	try {
		writer.flush();
	} catch (MutationsRejectedException e) {
		throw new RuntimeException(e);
	}
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
private void doScan(WebSocketClient client) throws Exception {
    long now = System.currentTimeMillis();
    String tableName = "qonduit.scanTest";
    Connector con = mac.getConnector(MAC_ROOT_USER, MAC_ROOT_PASSWORD);
    con.namespaceOperations().create("qonduit");
    con.tableOperations().create(tableName);
    BatchWriterConfig bwc = new BatchWriterConfig();
    bwc.setMaxLatency(2, TimeUnit.SECONDS);
    BatchWriter writer = con.createBatchWriter(tableName, bwc);

    ColumnVisibility cv = new ColumnVisibility();
    for (int i = 0; i < 10; i++) {
        Mutation m = new Mutation("m" + i);
        m.put("cf" + i, "cq" + i, cv, now + i, Integer.toString(i));
        writer.addMutation(m);
    }
    writer.flush();
    writer.close();
    sleepUninterruptibly(2, TimeUnit.SECONDS);
    List<byte[]> responses = new ArrayList<>();
    String id = UUID.randomUUID().toString();
    ScanRequest request = new ScanRequest();
    request.setRequestId(id);
    request.setTableName(tableName);
    request.setResultBatchSize(5);
    doIt(client, request, responses, 3);
    Assert.assertEquals(11, responses.size());
    for (byte[] b : responses) {
        KVPair kv = JsonSerializer.getObjectMapper().readValue(b, KVPair.class);
        Value val = kv.getValue();
        if (null != val) {
            int num = Integer.parseInt(new String(val.getValue()));
            Key key = kv.getKey().toKey();
            Assert.assertEquals("m" + num, key.getRow().toString());
            Assert.assertEquals("cf" + num, key.getColumnFamily().toString());
            Assert.assertEquals("cq" + num, key.getColumnQualifier().toString());
            Assert.assertEquals(now + num, key.getTimestamp());
            Assert.assertEquals(id, kv.getRequestId());
        } else {
            Assert.assertTrue(kv.isEndOfResults());
        }
    }
}
 

import org.apache.accumulo.core.client.BatchWriter; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts batchOpts = new BatchWriterOpts();
  opts.parseArgs(SignedConverterExample.class.getName(), args, batchOpts, scanOpts);

  Connector conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createDestinationTable) {
    conn.tableOperations().create(opts.destination);
  }

  // Transform entries
  Scanner scanner = conn.createScanner(opts.source, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);

  BatchWriterConfig bwConfig = batchOpts.getBatchWriterConfig();
  bwConfig.setDurability(opts.durability);
  BatchWriter writer = new SignedBatchWriter(conn, opts.destination, bwConfig, opts.signatureConfig, opts.signatureKeys);

  long count = 0;
  for (Entry<Key,Value> entry : scanner) {
    Mutation mutation = new Mutation(entry.getKey().getRow());
    mutation.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed(), entry.getKey()
        .getTimestamp(), entry.getValue());
    writer.addMutation(mutation);

    count++;
    if (count % 10000 == 0) {
      System.out.println(String.format("converted %d entries", count));
    }
  }

  writer.flush();
  writer.close();

  // delete table
  if (opts.deleteSourceTable)
    conn.tableOperations().delete(opts.source);
}