Java Scanner.setBatchSize方法代码示例

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);

  Connector conn = opts.getConnector();

  Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);
  BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());

  for (Entry<Key,Value> entry : scanner) {
    Key key = entry.getKey();
    Mutation m = new Mutation(key.getColumnQualifier());
    m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
    bw.addMutation(m);
  }

  bw.close();

}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private void execute(Opts opts, ScannerOpts scanOpts) throws Exception {
  conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createtable) {
    SortedSet<Text> partitionKeys = new TreeSet<>();
    for (int i = Byte.MIN_VALUE; i < Byte.MAX_VALUE; i++)
      partitionKeys.add(new Text(new byte[] {(byte) i}));
    conn.tableOperations().create(opts.getTableName());
    conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
  }

  // send mutations
  createEntries(opts);

  // read entries
  if (opts.readEntries) {
    // Note that the user needs to have the authorizations for the specified scan authorizations
    // by an administrator first
    Scanner scanner = new SignedScanner(conn, opts.getTableName(), opts.auths, opts.signatureConfig, opts.signatureKeys);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    for (Entry<Key,Value> entry : scanner)
      System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
  }

  // delete table
  if (opts.deletetable)
    conn.tableOperations().delete(opts.getTableName());
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private int findMaxDepth(Scanner scanner) {
  // do binary search to find max depth
  int origBatchSize = scanner.getBatchSize();
  scanner.setBatchSize(100);
  int depth = findMaxDepth(scanner, 0, 64, 999);
  scanner.setBatchSize(origBatchSize);
  return depth;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
public void run() throws Exception {

    entriesScanned = 0;
    inserts = 0;

    Scanner scanner = conn.createScanner(tableName, auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    BatchWriter bw = conn.createBatchWriter(tableName, bwOpts.getBatchWriterConfig());

    long t1 = System.currentTimeMillis();

    int depth = findMaxDepth(scanner);

    long t2 = System.currentTimeMillis();

    for (int d = depth; d > 0; d--) {
      calculateCounts(scanner, d, bw);
      // must flush so next depth can read what prev depth wrote
      bw.flush();
    }

    bw.close();

    long t3 = System.currentTimeMillis();

    System.out.printf("Max depth              : %d%n", depth);
    System.out.printf("Time to find max depth : %,d ms%n", (t2 - t1));
    System.out.printf("Time to compute counts : %,d ms%n", (t3 - t2));
    System.out.printf("Entries scanned        : %,d %n", entriesScanned);
    System.out.printf("Counts inserted        : %,d %n", inserts);
  }
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Gets a scanner over one row
 */
private static Scanner getRow(ScannerOpts scanOpts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
  // Create a scanner
  Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
  scanner.setBatchSize(scanOpts.scanBatchSize);
  // Say start key is the one with key of row
  // and end key is the one that immediately follows the row
  scanner.setRange(new Range(row));
  return scanner;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private void execute(Opts opts, ScannerOpts scanOpts) throws Exception {
  conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createtable) {
    SortedSet<Text> partitionKeys = new TreeSet<>();
    for (int i = Byte.MIN_VALUE; i < Byte.MAX_VALUE; i++)
      partitionKeys.add(new Text(new byte[] {(byte) i}));
    conn.tableOperations().create(opts.getTableName());
    conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
  }

  // send mutations
  createEntries(opts);

  // read entries
  if (opts.readEntries) {
    // Note that the user needs to have the authorizations for the specified scan authorizations
    // by an administrator first
    Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    for (Entry<Key,Value> entry : scanner)
      System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
  }

  // delete table
  if (opts.deletetable)
    conn.tableOperations().delete(opts.getTableName());
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts batchOpts = new BatchWriterOpts();
  opts.parseArgs(SignedConverterExample.class.getName(), args, batchOpts, scanOpts);

  Connector conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createDestinationTable) {
    conn.tableOperations().create(opts.destination);
  }

  // Transform entries
  Scanner scanner = conn.createScanner(opts.source, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);

  BatchWriterConfig bwConfig = batchOpts.getBatchWriterConfig();
  bwConfig.setDurability(opts.durability);
  BatchWriter writer = new SignedBatchWriter(conn, opts.destination, bwConfig, opts.signatureConfig, opts.signatureKeys);

  long count = 0;
  for (Entry<Key,Value> entry : scanner) {
    Mutation mutation = new Mutation(entry.getKey().getRow());
    mutation.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed(), entry.getKey()
        .getTimestamp(), entry.getValue());
    writer.addMutation(mutation);

    count++;
    if (count % 10000 == 0) {
      System.out.println(String.format("converted %d entries", count));
    }
  }

  writer.flush();
  writer.close();

  // delete table
  if (opts.deleteSourceTable)
    conn.tableOperations().delete(opts.source);
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts batchOpts = new BatchWriterOpts();
  opts.parseArgs(EncryptedConverterExample.class.getName(), args, batchOpts, scanOpts);

  Connector conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createDestinationTable) {
    conn.tableOperations().create(opts.destination);
  }

  // Transform entries
  Scanner scanner = conn.createScanner(opts.source, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);

  BatchWriterConfig bwConfig = batchOpts.getBatchWriterConfig();
  bwConfig.setDurability(opts.durability);
  BatchWriter writer = new EncryptedBatchWriter(conn, opts.destination, bwConfig, opts.encryptionConfig, opts.encryptionKeys);

  long count = 0;
  for (Entry<Key,Value> entry : scanner) {
    Mutation mutation = new Mutation(entry.getKey().getRow());
    mutation.put(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed(), entry.getKey()
        .getTimestamp(), entry.getValue());
    writer.addMutation(mutation);

    count++;
    if (count % 10000 == 0) {
      System.out.println(String.format("converted %d entries", count));
    }
  }

  writer.flush();
  writer.close();

  // delete table
  if (opts.deleteSourceTable)
    conn.tableOperations().delete(opts.source);
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
public Scanner createScannerForMetric(String sessionId, String metric, Map<String, String> tags, long startTime,
        long endTime, int lag, int scannerBatchSize, int scannerReadAhead) throws TimelyException {
    try {
        Authorizations auths = null;
        try {
            auths = getSessionAuthorizations(sessionId);
        } catch (NullPointerException npe) {
            // Session id being used for metric scanner, but session Id does
            // not exist in Auth Cache. Use Empty auths if anonymous access
            // allowed
            if (anonAccessAllowed) {
                auths = Authorizations.EMPTY;
            } else {
                throw npe;
            }
        }
        LOG.debug("Creating metric scanner for session: {} with auths: {}", sessionId, auths);
        Scanner s = connector.createScanner(this.metricsTable, auths);
        if (null == metric) {
            throw new IllegalArgumentException("metric name must be specified");
        }
        if (0 == startTime) {
            startTime = (System.currentTimeMillis() - getAgeOffForMetric(metric) - 1000);
            LOG.debug("Overriding zero start time to {} due to age off configuration", startTime);
        }
        byte[] start = MetricAdapter.encodeRowKey(metric, startTime);
        long endTimeStamp = (endTime == 0) ? (System.currentTimeMillis() - (lag * 1000)) : endTime;
        byte[] end = MetricAdapter.encodeRowKey(metric, endTimeStamp);
        s.setRange(new Range(new Text(start), true, new Text(end), false));
        SubQuery query = new SubQuery();
        query.setMetric(metric);
        if (null == tags) {
            tags = Collections.emptyMap();
        }
        query.setTags(tags);
        List<String> tagOrder = prioritizeTags(query);
        Map<String, String> orderedTags = orderTags(tagOrder, query.getTags());
        setQueryColumns(s, metric, orderedTags);
        s.setBatchSize(scannerBatchSize);
        s.setReadaheadThreshold(scannerReadAhead);
        return s;
    } catch (IllegalArgumentException | TableNotFoundException ex) {
        LOG.error("Error during lookup: " + ex.getMessage(), ex);
        throw new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), "Error during lookup: "
                + ex.getMessage(), ex.getMessage(), ex);
    }
}