Java FindIterable.batchSize方法代码示例

import com.mongodb.client.FindIterable; //导入方法依赖的package包/类
@Override
protected void collectVectors(Collection<String> terms, int limit) {
    Set<String> toFetch = terms.stream()
            .filter(t -> !this.vectorsCache.containsKey(t))
            .collect(Collectors.toSet());

    logger.debug("Cache has {} vectors, need to fetch more {}",
            terms.size() - toFetch.size(), toFetch.size());

    if (!toFetch.isEmpty()) {
        logger.info("Collecting {} term vectors from {}", toFetch.size(), dbName);
        FindIterable<Document> docs = getTermsColl().find(Filters.in(TERM_FIELD_NAME, toFetch));
        if (docs != null) {
            docs.batchSize(toFetch.size());
            for (Document doc : docs) {
                this.vectorsCache.put(doc.getString(TERM_FIELD_NAME), unmarshall(doc, limit));
            }
        }
    }
}
 

import com.mongodb.client.FindIterable; //导入方法依赖的package包/类
public void processData() {
    MongoCollection<Document> collection = this.database.getCollection("channels");
    logger.info("There are " + String.valueOf(collection.count()) + " channels!");

    FindIterable<Document> iterable = collection.find();
    iterable.batchSize(this.opts.getBatchSize());

    MongoCursor<Document> cursor = iterable.iterator();

    this.processor.run(cursor);
    cursor.close();
}
 

import com.mongodb.client.FindIterable; //导入方法依赖的package包/类
private void runnerUpdate(FindIterable<Document> iterable)
		throws SearchLibException, ClassNotFoundException, InstantiationException, IllegalAccessException,
		IOException, ParseException, SyntaxError, URISyntaxException, InterruptedException {
	final int limit = databaseCrawl.getBufferSize();
	iterable.batchSize(limit);
	final RestFieldMap fieldMap = (RestFieldMap) databaseCrawl.getFieldMap();
	List<IndexDocument> indexDocumentList = new ArrayList<IndexDocument>(0);
	LanguageEnum lang = databaseCrawl.getLang();
	FieldMapContext fieldMapContext = new FieldMapContext(client, lang);
	String uniqueField = client.getSchema().getUniqueField();
	MongoCursor<Document> cursor = iterable.iterator();
	while (cursor.hasNext() && !isAborted()) {

		String json = JSON.serialize(cursor.next());
		Object document = Configuration.defaultConfiguration().jsonProvider().parse(json);
		IndexDocument indexDocument = new IndexDocument(lang);
		fieldMap.mapJson(fieldMapContext, document, indexDocument);
		if (uniqueField != null && !indexDocument.hasContent(uniqueField)) {
			rwl.w.lock();
			try {
				ignoredDocumentCount++;
			} finally {
				rwl.w.unlock();
			}
			continue;
		}
		indexDocumentList.add(indexDocument);
		rwl.w.lock();
		try {
			pendingIndexDocumentCount++;
		} finally {
			rwl.w.unlock();
		}
		if (index(indexDocumentList, limit))
			setStatus(CrawlStatus.CRAWL);

	}
	index(indexDocumentList, 0);
}
 

import com.mongodb.client.FindIterable; //导入方法依赖的package包/类
private Function<Exchange, Object> createDoFindAll() {
    return exchange1 -> {
        Iterable<BasicDBObject> result;
        MongoCollection<BasicDBObject> dbCol = calculateCollection(exchange1);
        // do not use getMandatoryBody, because if the body is empty we want to retrieve all objects in the collection
        BasicDBObject query = null;
        // do not run around looking for a type converter unless there is a need for it
        if (exchange1.getIn().getBody() != null) {
            query = exchange1.getIn().getBody(BasicDBObject.class);
        }
        BasicDBObject fieldFilter = exchange1.getIn().getHeader(MongoDbConstants.FIELDS_FILTER, BasicDBObject.class);

        // get the batch size and number to skip
        Integer batchSize = exchange1.getIn().getHeader(MongoDbConstants.BATCH_SIZE, Integer.class);
        Integer numToSkip = exchange1.getIn().getHeader(MongoDbConstants.NUM_TO_SKIP, Integer.class);
        Integer limit = exchange1.getIn().getHeader(MongoDbConstants.LIMIT, Integer.class);
        BasicDBObject sortBy = exchange1.getIn().getHeader(MongoDbConstants.SORT_BY, BasicDBObject.class);
        FindIterable<BasicDBObject> ret;
        if (query == null && fieldFilter == null) {
            ret = dbCol.find(new BasicDBObject());
        } else if (fieldFilter == null) {
            ret = dbCol.find(query);
        } else {
            ret = dbCol.find(new BasicDBObject()).projection(fieldFilter);
        }

        if (sortBy != null) {
            ret.sort(sortBy);
        }

        if (batchSize != null) {
            ret.batchSize(batchSize);
        }

        if (numToSkip != null) {
            ret.skip(numToSkip);
        }

        if (limit != null) {
            ret.limit(limit);
        }

        if (!MongoDbOutputType.DBCursor.equals(endpoint.getOutputType())) {
            try {
                result = new ArrayList<>();
                ret.iterator().forEachRemaining(((List<BasicDBObject>) result)::add);
                exchange1.getOut().setHeader(MongoDbConstants.RESULT_PAGE_SIZE, ((List<BasicDBObject>) result).size());
            } finally {
                ret.iterator().close();
            }
        } else {
            result = ret;
        }
        return result;
    };
}