Java Scanner.fetchColumnFamily方法代码示例

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Get an {@link Iterator} over the {@link BindingSet}s that are stored in the PCJ table.
 *
 * @param accumuloConn - A connection to the Accumulo that hsots the PCJ table. (not null)
 * @param pcjTableName - The name of the PCJ table that will be scanned. (not null)
 * @param auths - the user's authorizations that will be used to scan the table. (not null)
 * @return An iterator over all of the {@link BindingSet}s that are stored as
 *   results for the PCJ.
 * @throws PCJStorageException The binding sets could not be fetched.
 */
public CloseableIterator<BindingSet> listResults(final Connector accumuloConn, final String pcjTableName, final Authorizations auths) throws PCJStorageException {
    requireNonNull(pcjTableName);

    // Fetch the Variable Orders for the binding sets and choose one of them. It
    // doesn't matter which one we choose because they all result in the same output.
    final PcjMetadata metadata = getPcjMetadata(accumuloConn, pcjTableName);
    final VariableOrder varOrder = metadata.getVarOrders().iterator().next();

    try {
        // Fetch only the Binding Sets whose Variable Order matches the selected one.
        final Scanner scanner = accumuloConn.createScanner(pcjTableName, auths);
        scanner.fetchColumnFamily( new Text(varOrder.toString()) );

        // Return an Iterator that uses that scanner.
        return new ScannerBindingSetIterator(scanner, varOrder);

    } catch (final TableNotFoundException e) {
        throw new PCJStorageException(String.format("PCJ Table does not exist for name '%s'.", pcjTableName), e);
    }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Scan accumulo for the results that are stored in a PCJ table. The
 * multimap stores a set of deserialized binding sets that were in the PCJ
 * table for every variable order that is found in the PCJ metadata.
 */
private static Multimap<String, BindingSet> loadPcjResults(final Connector accumuloConn, final String pcjTableName) throws PcjException, TableNotFoundException, BindingSetConversionException {
    final Multimap<String, BindingSet> fetchedResults = HashMultimap.create();

    // Get the variable orders the data was written to.
    final PcjTables pcjs = new PcjTables();
    final PcjMetadata pcjMetadata = pcjs.getPcjMetadata(accumuloConn, pcjTableName);

    // Scan Accumulo for the stored results.
    for(final VariableOrder varOrder : pcjMetadata.getVarOrders()) {
        final Scanner scanner = accumuloConn.createScanner(pcjTableName, new Authorizations());
        scanner.fetchColumnFamily( new Text(varOrder.toString()) );

        for(final Entry<Key, Value> entry : scanner) {
            final byte[] serializedResult = entry.getKey().getRow().getBytes();
            final BindingSet result = converter.convert(serializedResult, varOrder);
            fetchedResults.put(varOrder.toString(), result);
        }
    }

    return fetchedResults;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private void setFetchColumns(Scanner scanner, String fields[]) {
  fields = getFieldsToQuery(fields);
  for (String field : fields) {
    Pair<Text,Text> col = mapping.fieldMap.get(field);
    if (col != null) {
      if (col.getSecond() == null) {
        scanner.fetchColumnFamily(col.getFirst());
      } else {
        scanner.fetchColumn(col.getFirst(), col.getSecond());
      }
    } else {
      LOG.error("Mapping not found for field: " + field);
    }
  }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private long getRowCountFromTable(String tableName, Text signalColumn, Authorizations authorizations) {
    try {
        LOGGER.debug("BEGIN getRowCountFromTable(%s)", tableName);
        Scanner scanner = createScanner(tableName, null, authorizations);
        try {
            scanner.fetchColumnFamily(signalColumn);

            IteratorSetting countingIterator = new IteratorSetting(
                    100,
                    CountingIterator.class.getSimpleName(),
                    CountingIterator.class
            );
            scanner.addScanIterator(countingIterator);

            GRAPH_LOGGER.logStartIterator(scanner);

            long count = 0;
            for (Map.Entry<Key, Value> entry : scanner) {
                Long countForKey = LongCombiner.FIXED_LEN_ENCODER.decode(entry.getValue().get());
                LOGGER.debug("getRowCountFromTable(%s): %s: %d", tableName, entry.getKey().getRow(), countForKey);
                count += countForKey;
            }
            LOGGER.debug("getRowCountFromTable(%s): TOTAL: %d", tableName, count);
            return count;
        } finally {
            scanner.close();
        }
    } catch (TableNotFoundException ex) {
        throw new VertexiumException("Could not get count from table: " + tableName, ex);
    }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public CloseableIteration<Namespace, RyaDAOException> iterateNamespace() throws RyaDAOException {
    try {
        final Scanner scanner = connector.createScanner(tableLayoutStrategy.getNs(),
                ALL_AUTHORIZATIONS);
        scanner.fetchColumnFamily(INFO_NAMESPACE_TXT);
        final Iterator<Map.Entry<Key, Value>> result = scanner.iterator();
        return new AccumuloNamespaceTableIterator(result);
    } catch (final Exception e) {
        throw new RyaDAOException(e);
    }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    //read the params from AccumuloInputFormat
    Configuration conf = jobContext.getConfiguration();
    Instance instance = MRUtils.AccumuloProps.getInstance(jobContext);
    String user = MRUtils.AccumuloProps.getUsername(jobContext);
    AuthenticationToken password = MRUtils.AccumuloProps.getPassword(jobContext);
    String table = MRUtils.AccumuloProps.getTablename(jobContext);
    ArgumentChecker.notNull(instance);
    ArgumentChecker.notNull(table);

    //find the files necessary
    try {
        Connector connector = instance.getConnector(user, password);
        TableOperations tos = connector.tableOperations();
        String tableId = tos.tableIdMap().get(table);
        Scanner scanner = connector.createScanner("accumulo.metadata", Authorizations.EMPTY); //TODO: auths?
        scanner.setRange(new Range(new Text(tableId + "\u0000"), new Text(tableId + "\uFFFD")));
        scanner.fetchColumnFamily(new Text("file"));
        List<String> files = new ArrayList<String>();
        List<InputSplit> fileSplits = new ArrayList<InputSplit>();
        for (Map.Entry<Key, Value> entry : scanner) {
            String file = entry.getKey().getColumnQualifier().toString();
            Path path = new Path(file);
            FileSystem fs = path.getFileSystem(conf);
            FileStatus fileStatus = fs.getFileStatus(path);
            long len = fileStatus.getLen();
            BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
            files.add(file);
            fileSplits.add(new FileSplit(path, 0, len, fileBlockLocations[0].getHosts()));
        }
        System.out.println(files);
        return fileSplits;
    } catch (Exception e) {
        throw new IOException(e);
    }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Checks to see if the provided term appears in other documents.
 * @param term the term to search for.
 * @param currentDocId the current document ID that the search term exists in.
 * @return {@code true} if the term was found in other documents. {@code false} otherwise.
 */
private boolean doesTermExistInOtherDocs(final String term, final int currentDocId, final Text docIdText) {
    try {
        final String freeTextDocTableName = getFreeTextDocTablename(conf);
        final Scanner scanner = getScanner(freeTextDocTableName);

        final String t = StringUtils.removeEnd(term, "*").toLowerCase();
        final Text queryTerm = ColumnPrefixes.getTermColFam(t);

        // perform query and read results
        scanner.fetchColumnFamily(queryTerm);

        for (final Entry<Key, Value> entry : scanner) {
            final Key key = entry.getKey();
            final Text row = key.getRow();
            final int rowId = Integer.parseInt(row.toString());
            // We only want to check other documents from the one we're deleting
            if (rowId != currentDocId) {
                final Text columnFamily = key.getColumnFamily();
                final String columnFamilyValue = columnFamily.toString();
                // Check that the value has the term prefix
                if (columnFamilyValue.startsWith(ColumnPrefixes.TERM_CF_PREFIX.toString())) {
                    final Text text = ColumnPrefixes.removePrefix(columnFamily);
                    final String value = text.toString();
                    if (value.equals(term)) {
                        return true;
                    }
                }
            }
        }
    } catch (final IOException e) {
        logger.error("Error searching for the existance of the term in other documents", e);
    }
    return false;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
@Override
public CloseableIterator<BindingSet> listResults(final String queryId, final Optional<Long> binId)
        throws PeriodicQueryStorageException {
    requireNonNull(queryId);

    final String tableName = tableNameFactory.makeTableName(ryaInstance, queryId);
    // Fetch the Variable Orders for the binding sets and choose one of
    // them. It
    // doesn't matter which one we choose because they all result in the
    // same output.
    final PeriodicQueryStorageMetadata metadata = getPeriodicQueryMetadata(queryId);
    final VariableOrder varOrder = metadata.getVariableOrder();

    try {
        // Fetch only the Binding Sets whose Variable Order matches the
        // selected one.
        final Scanner scanner = accumuloConn.createScanner(tableName, auths);
        scanner.fetchColumnFamily(new Text(varOrder.toString()));
        if (binId.isPresent()) {
            scanner.setRange(Range.prefix(getRowPrefix(binId.get())));
        }
        return new AccumuloValueBindingSetIterator(scanner);

    } catch (final Exception e) {
        throw new PeriodicQueryStorageException(String.format("PCJ Table does not exist for name '%s'.", tableName), e);
    }
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Queries metadata table to determine which terms are indexed.
 * 
 * @param c
 * @param auths
 * @param queryLiterals
 * @param datatypes
 *          - optional list of types
 * @return map of indexed field names to types to normalizers used in this date range
 * @throws TableNotFoundException
 * @throws IllegalAccessException
 * @throws InstantiationException
 */
protected Map<String,Multimap<String,Class<? extends Normalizer>>> findIndexedTerms(Connector c, Authorizations auths, Set<String> queryLiterals,
    Set<String> datatypes) throws TableNotFoundException, InstantiationException, IllegalAccessException {
  
  Map<String,Multimap<String,Class<? extends Normalizer>>> results = new HashMap<String,Multimap<String,Class<? extends Normalizer>>>();
  
  for (String literal : queryLiterals) {
    if (log.isDebugEnabled())
      log.debug("Querying " + this.getMetadataTableName() + " table for " + literal);
    Range range = new Range(literal.toUpperCase());
    Scanner scanner = c.createScanner(this.getMetadataTableName(), auths);
    scanner.setRange(range);
    scanner.fetchColumnFamily(new Text(WikipediaMapper.METADATA_INDEX_COLUMN_FAMILY));
    for (Entry<Key,Value> entry : scanner) {
      if (!results.containsKey(literal)) {
        Multimap<String,Class<? extends Normalizer>> m = HashMultimap.create();
        results.put(literal, m);
      }
      // Get the column qualifier from the key. It contains the datatype and normalizer class
      String colq = entry.getKey().getColumnQualifier().toString();
      if (null != colq && colq.contains("\0")) {
        int idx = colq.indexOf("\0");
        if (idx != -1) {
          String type = colq.substring(0, idx);
          // If types are specified and this type is not in the list then skip it.
          if (null != datatypes && !datatypes.contains(type))
            continue;
          try {
            @SuppressWarnings("unchecked")
            Class<? extends Normalizer> clazz = (Class<? extends Normalizer>) Class.forName(colq.substring(idx + 1));
            if (!normalizerCacheMap.containsKey(clazz))
              normalizerCacheMap.put(clazz, clazz.newInstance());
            results.get(literal).put(type, clazz);
          } catch (ClassNotFoundException e) {
            log.error("Unable to find normalizer on class path: " + colq.substring(idx + 1), e);
            results.get(literal).put(type, LcNoDiacriticsNormalizer.class);
          }
        } else {
          log.warn("EventMetadata entry did not contain NULL byte: " + entry.getKey().toString());
        }
      } else {
        log.warn("ColumnQualifier null in EventMetadata for key: " + entry.getKey().toString());
      }
    }
  }
  if (log.isDebugEnabled())
    log.debug("METADATA RESULTS: " + results.toString());
  return results;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
private Map<String, AccumuloTablet> createTabletMap(final Connector connector, final String user,
                                                    final String tableId)
        throws TableNotFoundException, AccumuloSecurityException, AccumuloException {
    // The table accumulo.metadata has the following form (information taken from
    // {@link https://accumulo.apache.org/1.8/accumulo_user_manual.html#metadata}).
    // - Every tablet has its own row
    // - Every row starts with the table id followed by ';' or '<' and ends with the end row split point for that tablet
    // - If the CF is 'file' then the CQ is the name of an RFile and the value contains the size of the file in bytes
    // and the number of key-value pairs in that file, separated by a ','

    // Create scanner (not batchscanner as we need the results to be in order) to read the rows of the
    // accumulo.metadata table relevant to table with id 'tableId'. As we only need file information, set the
    // CF to 'file'.
    LOGGER.info("Scanning accumulo.metadata table");
    final Authorizations auths = connector.securityOperations().getUserAuthorizations(user);
    final Scanner scanner = connector.createScanner("accumulo.metadata", auths);
    scanner.setRange(new Range(new Text(tableId), true, new Text(tableId + "<"), true));
    scanner.fetchColumnFamily(new Text("file"));

    final Map<String, AccumuloTablet> tabletMap = new HashMap<>();
    final Iterator<Map.Entry<Key, Value>> iterator = scanner.iterator();
    int idx = 0;
    String lastTablet = null;
    String currentStart = null;
    if (!iterator.hasNext()) {
        LOGGER.warn("No Rfiles found");
    }
    while (iterator.hasNext()) {
        final Map.Entry<Key, Value> entry = iterator.next();
        final Key key = entry.getKey();

        // Row id is tableId;splitPoint
        // Last tablet is tableId<
        final String[] fields = key.getRow().toString().split(";");
        final String tabletName;
        if (fields.length == 2) {
            tabletName = fields[0];
        } else if (fields.length == 1) {
            tabletName = LAST_TABLET;
        } else {
            throw new RuntimeException("Row in accumulo.metadata didn't have the expected number of fields: "
            + "Expected 1 or 2, got " + fields.length);
        }

        // Detect start of a new tablet
        if (!tabletName.equals(lastTablet)) {
            currentStart = lastTablet;
            lastTablet = tabletName;
        }

        final String currentEnd;
        if (tabletName.equals(LAST_TABLET)) {
            currentEnd = null;
        } else {
            currentEnd = tabletName;
        }

        if (!tabletMap.containsKey(tabletName)) {
            tabletMap.put(tabletName, new AccumuloTablet(super.id(), idx, currentStart, currentEnd));
            idx += 1;
        }
        final AccumuloTablet tablet = tabletMap.get(tabletName);

        final String rFile = key.getColumnQualifier().toString();
        tablet.addRFile(rFile);
        LOGGER.info("Tablet {} has rFile {}", tabletName, rFile);
    }

    return tabletMap;
}
 

import org.apache.accumulo.core.client.Scanner; //导入方法依赖的package包/类
/**
 * Get a list of timestamps that represents all of the Prospect runs that have
 * ever been performed.
 *
 * @param auths - The authorizations used to scan the table for prospects.
 * @return A list of timestamps representing each Prospect run that was found.
 * @throws TableNotFoundException The table name that was provided when this
 *   class was constructed does not match a table that the connector has access to.
 */
public Iterator<Long> getProspects(String[] auths) throws TableNotFoundException {
    final Scanner scanner = connector.createScanner(tableName, new Authorizations(auths));
    scanner.setRange(Range.exact(METADATA));
    scanner.fetchColumnFamily(new Text(PROSPECT_TIME));

    return new ProspectTimestampIterator( scanner.iterator() );
}