Java ScanResult.getLastEvaluatedKey方法代码示例

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
protected PageResults<Map<String, AttributeValue>> fetchPage(RequestLimit lim) {
  // Read from DynamoDB
  RetryResult<ScanResult> retryResult = context.getClient().scanTable(tableName, null, segment,
      context.getSplit().getTotalSegments(), lastEvaluatedKey, lim.items, context.getReporter());

  ScanResult result = retryResult.result;
  int retries = retryResult.retries;

  double consumedCapacityUnits = 0.0;
  if (result.getConsumedCapacity() != null) {
    consumedCapacityUnits = result.getConsumedCapacity().getCapacityUnits();
  }
  return new PageResults<>(result.getItems(), result.getLastEvaluatedKey(), consumedCapacityUnits,
      retries);
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@SuppressFBWarnings(value = "IT_NO_SUCH_ELEMENT",
    justification = "https://github.com/awslabs/dynamodb-janusgraph-storage-backend/issues/222")
@Override
public ScanResult next() {
    final Scan backoff = new Scan(request, delegate, lastConsumedCapacity);
    ScanResult result = null;
    try {
        result = backoff.runWithBackoff(); //this will be non-null or runWithBackoff throws
    } catch (BackendException e) {
        throw new BackendRuntimeException(e);
    }

    if (result.getConsumedCapacity() != null) {
        lastConsumedCapacity = result.getConsumedCapacity().getCapacityUnits().intValue();
    }

    if (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    return result;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private long getDynamoItemCount() {

		ScanResult scanResult = dynamoClient.scan(TEST_DYNAMO_TABLE_NAME, Lists.newArrayList("Uuid"));
		long itemCount = scanResult.getCount();

		while(scanResult.getLastEvaluatedKey() != null && !scanResult.getLastEvaluatedKey().isEmpty()) {
			scanResult = dynamoClient.scan(new ScanRequest().withTableName(TEST_DYNAMO_TABLE_NAME).withAttributesToGet("Uuid").withExclusiveStartKey(scanResult.getLastEvaluatedKey()));
			itemCount += scanResult.getCount();
		}

		return itemCount;
	}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private Stream<Entry> scan(SecretEventStream.Filter<Entry> filter, Converters converters) {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    FilterGenerator filterGenerator = new FilterGenerator();
    FilterGenerator.Filter generated = filterGenerator.process(filter.parsedAttributeCondition.get(), converters);

    if(!generated.expressionAttributeNames.isEmpty()) {
        scanRequest.withExpressionAttributeNames(generated.expressionAttributeNames);
    }

    if (!generated.expressionAttributeValues.isEmpty()) {
        scanRequest.withExpressionAttributeValues(generated.expressionAttributeValues);
    }

    scanRequest.withFilterExpression(generated.filterExpression);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }

    Stream<Entry> typedResult = results.stream().map(this::fromMap);

    if (filter.reverse) {
        typedResult = Lists.reverse(typedResult.collect(Collectors.toCollection(LinkedList::new))).stream();
    }

    return typedResult;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private ScanOutcome scan(ScanSpec scanSpec, PageIterator pageIterator) {
    if ( null == _convertMarker ) {
        throw new IllegalStateException("Index must first be initialized with ConvertMarker");
    }
    if ( pageIterator.getPageSize() <= 0 ) {
        return new ScanOutcome(new ScanResult());
    }
    ItemCollection<ScanOutcome> itemCollection =
        maybeBackoff(true, () -> _scan.scan(withMarker(scanSpec, pageIterator)));

    if ( null != itemCollection ) {
        Iterator<Page<Item, ScanOutcome>> iterator = itemCollection.pages().iterator();
        if ( iterator.hasNext() ) {
            ScanOutcome outcome = maybeBackoff(true, () -> iterator.next().getLowLevelResult());
            ScanResult result = outcome.getScanResult();
            Map<String,AttributeValue> lastKey = result.getLastEvaluatedKey();
            if ( null != lastKey && ! lastKey.isEmpty() ) {
                pageIterator.setMarker(toMarker(lastKey, false));
            } else {
                pageIterator.setMarker(null);
            }
            return outcome;
        }
    }
    pageIterator.setMarker(null);
    return new ScanOutcome(new ScanResult());
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
public SegmentedScanResult call() {
    ScanResult result = null;
    result = runWithBackoff();

    final ConsumedCapacity cc = result.getConsumedCapacity();

    if (cc != null && cc.getCapacityUnits() != null) {
        lastConsumedCapacity = result.getConsumedCapacity()
                .getCapacityUnits().intValue();
    } else if (result.getScannedCount() != null && result.getCount() != null) {

        final boolean isConsistent = request.getConsistentRead();
        int itemSize = isConsistent ? BootstrapConstants.STRONGLY_CONSISTENT_READ_ITEM_SIZE
                : BootstrapConstants.EVENTUALLY_CONSISTENT_READ_ITEM_SIZE;

        lastConsumedCapacity = (result.getScannedCount() / (int) Math.max(1.0, result.getCount()))
                * (ItemSizeCalculator.calculateScanResultSizeInBytes(result) / itemSize);
    }

    if (result.getLastEvaluatedKey() != null
            && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    if (lastConsumedCapacity > 0) {
        rateLimiter.acquire(lastConsumedCapacity);
    }
    return new SegmentedScanResult(result, request.getSegment());
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
public void run() {
    Map<String, AttributeValue> exclusiveStartKey = null;
    ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withAttributesToGet(attributesToGet)
            .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withTotalSegments(numOfSegments).withSegment(segmentNum);
    boolean scanNumLimitReached = false;
    while (!scanNumLimitReached) {
        scanRequest.withExclusiveStartKey(exclusiveStartKey);
        ScanResult scanResult = dynamoDBClient.scan(scanRequest);
        if(!isRunningOnDDBLocal) {
            // DDB Local does not support rate limiting
            tableReadRateLimiter.adjustRateWithConsumedCapacity(scanResult.getConsumedCapacity());
        }

        for (Map<String, AttributeValue> item : scanResult.getItems()) {
            checkItemViolationAndAddDeleteRequest(item);
            itemsScanned.addAndGet(1);
            itemScannedByThread += 1;
            scanNumLimitReached = isScanNumberLimitReached();
            if(scanNumLimitReached) {
                break;
            }
        }

        if (deleteViolationAfterFound) {
            sendDeleteViolations();
        }
        PrintHelper.printScanProgress(itemsScanned.get(), itemScannedByThread, violationFoundByThread, violationDeleteByThread);

        if (null == (exclusiveStartKey = scanResult.getLastEvaluatedKey())) {
            break;
        }
    }
    return;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
public List<Map<String, AttributeValue>> getItems(String tableName) {
    List<Map<String, AttributeValue>> items = new ArrayList<Map<String, AttributeValue>>();
    ScanRequest scanRequest = new ScanRequest().withTableName(tableName);
    ScanResult scanResult = client.scan(scanRequest);
    Map<String, AttributeValue> lastEvaluatedKey = null;
    do {
        scanRequest = new ScanRequest().withTableName(tableName).withExclusiveStartKey(lastEvaluatedKey);
        scanResult = client.scan(scanRequest);
        items.addAll(scanResult.getItems());
        lastEvaluatedKey = scanResult.getLastEvaluatedKey();
    } while(lastEvaluatedKey != null);
    return items;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
/**
 * Reads a page from a standard DynamoDB table.
 * @param <P> type of object
 * @param appid the app identifier (name)
 * @param p a {@link Pager}
 * @return the last row key of the page, or null.
 */
public static <P extends ParaObject> List<P> readPageFromTable(String appid, Pager p) {
	Pager pager = (p != null) ? p : new Pager();
	ScanRequest scanRequest = new ScanRequest().
			withTableName(getTableNameForAppid(appid)).
			withLimit(pager.getLimit()).
			withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);

	if (!StringUtils.isBlank(pager.getLastKey())) {
		scanRequest = scanRequest.withExclusiveStartKey(Collections.
				singletonMap(Config._KEY, new AttributeValue(pager.getLastKey())));
	}

	ScanResult result = getClient().scan(scanRequest);
	LinkedList<P> results = new LinkedList<>();
	for (Map<String, AttributeValue> item : result.getItems()) {
		P obj = fromRow(item);
		if (obj != null) {
			results.add(obj);
		}
	}

	if (result.getLastEvaluatedKey() != null) {
		pager.setLastKey(result.getLastEvaluatedKey().get(Config._KEY).getS());
	} else if (!results.isEmpty()) {
		// set last key to be equal to the last result - end reached.
		pager.setLastKey(results.peekLast().getId());
	}
	return results;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private void fetchIndexList() {
   final ScanRequest req = new ScanRequest("s3-index-example.mikedeck-index").withAttributesToGet("Key");
   ScanResult result = DDB.scan(req);
   addIndexItems(result.getItems());
   while (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
      req.setExclusiveStartKey(result.getLastEvaluatedKey());
      result = DDB.scan(req);
      addIndexItems(result.getItems());
   }
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
    String tableName = storeToTableName(storeName);
    ScanRequest scanRequest = new ScanRequest(tableName);
    scanRequest.setAttributesToGet(Arrays.asList(ROW_KEY_ATTR_NAME)); // attributes to get
    if (continuationToken != null) {
        scanRequest.setExclusiveStartKey(makeDDBKey(continuationToken));
    }
    List<String> rowKeys = new ArrayList<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) {
            break;
        }
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get(ROW_KEY_ATTR_NAME);
            rowKeys.add(rowAttr.getS());
            if (rowKeys.size() >= count) {
                break;
            }
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) {
            break;
        }
        scanRequest.setExclusiveStartKey(lastEvaluatedKey);
    }
    return rowKeys;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
	Timer t = new Timer();
	ScanRequest scanRequest = new ScanRequest(getTenant().getName());
    scanRequest.setAttributesToGet(Arrays.asList("key")); // attributes to get
    //if (continuationToken != null) {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_" + continuationToken, "\u007F"));
    //} else {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_", "\0"));
    //}

    Set<String> rowKeys = new HashSet<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = m_client.scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) break;
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get("key");
            if(!rowAttr.getS().startsWith(storeName)) continue;
            String name = rowAttr.getS().substring(storeName.length() + 1);
            if(continuationToken != null && continuationToken.compareTo(name) >= 0) continue;
            rowKeys.add(name);
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) break;
    	scanRequest.setExclusiveStartKey(getPrimaryKey(lastEvaluatedKey.get("key").getS(), "\u007F"));
    }
    List<String> list = new ArrayList<>(rowKeys);
    Collections.sort(list);
	m_logger.debug("get rows in {} in {}", storeName, t);
    return list;
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while(true) {
            ScanRequest scanRequest = new ScanRequest()
                .withTableName(tableName)
                .withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey)
                .withTotalSegments(totalSegments)
                .withSegment(segment);
            
            ScanResult result = client.scan(scanRequest);
            
            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();
            
            // print items returned from scan request
            processScanResult(segment, result);
            
            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    } catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    } finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of " + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}
 

import com.amazonaws.services.dynamodbv2.model.ScanResult; //导入方法依赖的package包/类
private Stream<Entry> all() {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }


    return results.stream().map(this::fromMap);
}