Java CursorMarkParams类代码示例

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
/**
 * Loop over all the documents returned by the query. This method queries the DB multiple times. Every time we get
 * data back, we pass it onto a processor, and stop processing data if the processor tells us it's had enough.
 *
 * @param server    the solr db
 * @param q         the query
 * @param uniqueKey the solr uniqueKey field to sort on. Required for solr's Cursor functionality.
 *@param processor the processor to handle the data. If the function returns true, we stop fetching more data.
 *  @throws IOException
 * @throws SolrServerException
 */
static void processAllDocs(SolrClient server, SolrQuery q,
                           String uniqueKey, Function<Collection<SolrDocument>, Boolean> processor
) throws IOException, SolrServerException {
    boolean done = false;
    String oldCursorMark;
    String cursorMark = CursorMarkParams.CURSOR_MARK_START;
    QueryResponse resp;

    // Cursor functionality requires a sort containing a uniqueKey field tie breaker
    q.addSort(uniqueKey, SolrQuery.ORDER.desc);

    while (!done) {
        oldCursorMark = cursorMark;
        q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
        resp = server.query(q);
        done = processor.apply(resp.getResults());
        cursorMark = resp.getNextCursorMark();
        done = done || oldCursorMark.equals(cursorMark);
    }
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
protected MCROAISolrResult solrQuery(Optional<String> cursor) throws SolrServerException, IOException {
    SolrQuery query = getBaseQuery(CommonParams.Q);

    // set support
    if (this.set != null) {
        String setId = this.set.getSetId();
        MCROAISetConfiguration<SolrQuery, SolrDocument, String> setConfig = getSetManager().getConfig(setId);
        setConfig.getHandler().apply(this.set, query);
    }
    // from & until
    if (this.from != null || this.until != null) {
        String fromUntilCondition = buildFromUntilCondition(this.from, this.until);
        query.add(CommonParams.FQ, fromUntilCondition);
    }

    // cursor
    query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursor.orElse(CursorMarkParams.CURSOR_MARK_START));
    query.set(CommonParams.ROWS, String.valueOf(getPartitionSize()));
    query.set(CommonParams.SORT, "id asc");

    // do the query
    SolrClient solrClient = MCRSolrClientFactory.getSolrClient();
    QueryResponse response = solrClient.query(query);
    Collection<MCROAISetResolver<String, SolrDocument>> setResolver = getSetResolver(response.getResults());
    return new MCROAISolrResult(response, d -> toHeader(d, setResolver));
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
/**
 * @see DATASOLR-162
 */
@Test
public void shouldNotModifyInitialQueryWhenRequestingResults() {

	SolrQuery initialQuery = new SolrQuery("*:*");
	DelegatingCursorFake<String> cursor = new DelegatingCursorFake<String>(initialQuery, null);
	cursor.open();

	SolrQuery executedQuey = cursor.getLastUsedQuery();

	assertThat(executedQuey, not(equalTo(initialQuery)));
	assertThat(executedQuey.getQuery(), equalTo(initialQuery.getQuery()));
	assertThat(executedQuey.get(CursorMarkParams.CURSOR_MARK_PARAM), equalTo(CursorMarkParams.CURSOR_MARK_START));

	assertThat(initialQuery.get(CursorMarkParams.CURSOR_MARK_PARAM), nullValue());
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
private void regularFinishStage(ResponseBuilder rb) {
  // We may not have been able to retrieve all the docs due to an
  // index change.  Remove any null documents.
  for (Iterator<SolrDocument> iter = rb._responseDocs.iterator(); iter.hasNext();) {
    if (iter.next() == null) {
      iter.remove();
      rb._responseDocs.setNumFound(rb._responseDocs.getNumFound()-1);
    }
  }

  rb.rsp.add("response", rb._responseDocs);
  if (null != rb.getNextCursorMark()) {
    rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, 
               rb.getNextCursorMark().getSerializedTotem());
  }
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
@Override
public SolrQuery transform(final Query query) {
    final SolrQuery solrQuery = new SolrQuery(query.getQuery());
    solrQuery.setParam("q.op", query.getDefaultOperator().name());

    if (query.getCursorMark() != null) {
        solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, query.getCursorMark());
    } else {
        solrQuery.setStart(query.getOffset());
    }

    solrQuery.setRows(query.getRows());

    if (query.getReturnFields() != null) {
        query.getReturnFields().stream().forEach(f -> solrQuery.addField(f.getName()));
    }

    if (query.getHighlightFields() != null && !query.getHighlightFields().isEmpty()) {
        solrQuery.setHighlight(true);
        query.getHighlightFields().stream().forEach(hf -> solrQuery.addHighlightField(hf.getName()));
    }

    if (query.getFacetFields() != null) {
        query.getFacetFields().stream().forEach(ff -> solrQuery.addFacetField(ff.getName()));
    }

    if (query.getSorts() != null) {
        for (Sort sort : query.getSorts()) {
            SolrQuery.ORDER solrOrder = sort.getSortOrder() == SortOrder.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
            SolrQuery.SortClause sortClause = new SolrQuery.SortClause(sort.getField().getName(), solrOrder);
            solrQuery.addSort(sortClause);
        }
    }

    if (query.getFilterQueries() != null) {
        query.getFilterQueries().stream().forEach(fq -> solrQuery.addFilterQuery(fq.getField().getName() + ":" + fq.getValue()));
    }

    if (query.getParams() != null) {
        query.getParams().entrySet().stream().forEach(e -> solrQuery.add(e.getKey(), e.getValue()));
    }

    return solrQuery;
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
private SolrQuery getQueryParams(BoundedSolrSource source) {
  String query = source.spec.getQuery();
  if (query == null) {
    query = "*:*";
  }
  SolrQuery solrQuery = new SolrQuery(query);
  solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
  solrQuery.setRows(source.spec.getBatchSize());
  solrQuery.addSort(uniqueKey, SolrQuery.ORDER.asc);
  if (source.replica != null) {
    solrQuery.setDistrib(false);
  }
  return solrQuery;
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
protected DelegatingCursor(SolrQuery query, String initalCursorMark) {

		this.referenceQuery = query;
		this.cursorMark = StringUtils.hasText(initalCursorMark) ? initalCursorMark : CursorMarkParams.CURSOR_MARK_START;
		this.state = State.REDAY;
		this.delegate = Collections.<T> emptyList().iterator();
	}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
private void load(String cursorMark) {

		SolrQuery query = referenceQuery.getCopy();
		query.set(CursorMarkParams.CURSOR_MARK_PARAM, this.getCursorMark());

		PartialResult<T> result = doLoad(query);
		process(result);
	}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
private void processQuery(SolrQuery query, WorkLog workLog) throws SolrServerException, IOException {
  log.debug("Query for rule : {}", query.toString());
  Timer.Context context = getTimer(getName() + ".processQuery").time();
  // need to commit here so that we can ignore documents just processed
  client.commit();

  boolean more = true;
  String cursor = CursorMarkParams.CURSOR_MARK_START;
  while (more) {
    query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursor);
    Timer.Context contextQuery = getTimer(getName() + ".query").time();

    QueryResponse response = client.query(query);
    workLog.ranSearch();
    SolrDocumentList results = response.getResults();
    log.debug("Found {} (of {} docs) in QT = {} ms", results.size(), results.getNumFound(), response.getQTime());
    String nextCursor = response.getNextCursorMark();
    if (nextCursor == null || cursor.equals(nextCursor)) {
      more = false;
    }
    distributeResponse(results, workLog);
    cursor = nextCursor;
    contextQuery.stop();
  }

  // We do this at a higher level too, so this would seem redundant. There is a trade-off. Allowing parallelism
  // between rules means rules can sometimes be re-processed redundantly. The higher level waitUntilCaughtUp() will
  // ensure we never process rules at the same time rules are being changed.
  // By doing a wait here as well however, we can collect accurate statistics about how much actual write activity we
  // are really generating by passing the workLog into the work pool.
  // When we have a better awareness of the typical work patterns it might be worth disabling this method call and
  // then stop collecting the metrics to improve throughput.
  waitUntilCaughtUp();
  context.stop();
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
@Override
public boolean hasNext() {
	try {
		final QueryResponse response = solr.query(query);
		sentCursorMark = query.get(CursorMarkParams.CURSOR_MARK_PARAM);
		nextCursorMark = response.getNextCursorMark();
		page = response.getResults();
		return !page.isEmpty();
	} catch (final Exception exception) {
		throw new RuntimeException(exception);
	}
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
/**
 * Builds a new iterator with the given data.
 * 
 * @param solr the SOLR facade.
 * @param query the query that will be submitted.
 */
SolrDeepPagingIterator(final SolrServer solr, final SolrQuery query) {
	this.solr = solr;
	this.query = query;
	this.sentCursorMark = CursorMarkParams.CURSOR_MARK_START;
	this.query.set(CursorMarkParams.CURSOR_MARK_PARAM, CursorMarkParams.CURSOR_MARK_START);
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
@Test
public void testPagingWithCursorMark() throws QueryException {
    int pageSize = 2;
    String cursorMark = CursorMarkParams.CURSOR_MARK_START;

    boolean done = false;
    List<String> ids = new ArrayList<>();

    // page through until the returned cursorMark is equal to the previous cursorMark

    while (!done) {
        // required to include a sort on the uniqueKey field to use cursorMark
        Query query = new ExampleSummaryQuery("*:*", cursorMark, pageSize);
        query.addSort(ExampleField.CREATE_DATE, SortOrder.DESC);
        query.addSort(ExampleField.ID, SortOrder.ASC);

        QueryResults<ExampleSummary> results = queryService.search(query);

        assertNotNull(results);
        assertNotNull(results.getResults());
        assertTrue(results.getResults().size() <= pageSize);
        assertEquals(5, results.getTotalResults());
        assertEquals(0, results.getOffset());
        assertEquals(pageSize, results.getPageSize());
        assertNotNull(results.getCursorMark());

        results.getResults().stream().forEach(r -> ids.add(r.getId()));

        if (results.getCursorMark().equals(cursorMark)) {
            done = true;
        }

        cursorMark = results.getCursorMark();
    }

    // verify we got ids 1-5 in order
    assertEquals(5, ids.size());
    for (int i=0; i < 5; i++) {
        assertEquals(String.valueOf(5-i), ids.get(i));
    }

}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
private BoundedSolrReader(BoundedSolrSource source) {
  this.source = source;
  this.cursorMark = CursorMarkParams.CURSOR_MARK_START;
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
@Override
public void setResponse( NamedList<Object> res )
{
  super.setResponse( res );
  
  // Look for known things
  for( int i=0; i<res.size(); i++ ) {
    String n = res.getName( i );
    if( "responseHeader".equals( n ) ) {
      _header = (NamedList<Object>) res.getVal( i );
    }
    else if( "response".equals( n ) ) {
      _results = (SolrDocumentList) res.getVal( i );
    }
    else if( "sort_values".equals( n ) ) {
      _sortvalues = (NamedList<ArrayList>) res.getVal( i );
    }
    else if( "facet_counts".equals( n ) ) {
      _facetInfo = (NamedList<Object>) res.getVal( i );
      // extractFacetInfo inspects _results, so defer calling it
      // in case it hasn't been populated yet.
    }
    else if( "debug".equals( n ) ) {
      _debugInfo = (NamedList<Object>) res.getVal( i );
      extractDebugInfo( _debugInfo );
    }
    else if( "grouped".equals( n ) ) {
      _groupedInfo = (NamedList<Object>) res.getVal( i );
      extractGroupedInfo( _groupedInfo );
    }
    else if("expanded".equals(n)) {
      _expandedResults = (Map<String, SolrDocumentList>) res.getVal( i );
    }
    else if( "highlighting".equals( n ) ) {
      _highlightingInfo = (NamedList<Object>) res.getVal( i );
      extractHighlightingInfo( _highlightingInfo );
    }
    else if ( "spellcheck".equals( n ) )  {
      _spellInfo = (NamedList<NamedList<Object>>) res.getVal( i );
      extractSpellCheckInfo( _spellInfo );
    }
    else if ( "stats".equals( n ) )  {
      _statsInfo = (NamedList<Object>) res.getVal( i );
      extractStatsInfo( _statsInfo );
    }
    else if ( "terms".equals( n ) ) {
      _termsInfo = (NamedList<NamedList<Number>>) res.getVal( i );
      extractTermsInfo( _termsInfo );
    }
    else if ( CursorMarkParams.CURSOR_MARK_NEXT.equals( n ) ) {
      _cursorMarkNext = (String) res.getVal( i );
    }
  }
  if(_facetInfo != null) extractFacetInfo( _facetInfo );
}
 

import org.apache.solr.common.params.CursorMarkParams; //导入依赖的package包/类
@Override
public void prepare(ResponseBuilder rb) throws IOException
{

  SolrQueryRequest req = rb.req;
  SolrParams params = req.getParams();
  if (!params.getBool(COMPONENT_NAME, true)) {
    return;
  }
  SolrQueryResponse rsp = rb.rsp;

  // Set field flags    
  ReturnFields returnFields = new SolrReturnFields( req );
  rsp.setReturnFields( returnFields );
  int flags = 0;
  if (returnFields.wantsScore()) {
    flags |= SolrIndexSearcher.GET_SCORES;
  }
  rb.setFieldFlags( flags );

  String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);

  // get it from the response builder to give a different component a chance
  // to set it.
  String queryString = rb.getQueryString();
  if (queryString == null) {
    // this is the normal way it's set.
    queryString = params.get( CommonParams.Q );
    rb.setQueryString(queryString);
  }

  try {
    QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
    Query q = parser.getQuery();
    if (q == null) {
      // normalize a null query to a query that matches nothing
      q = new BooleanQuery();        
    }
    rb.setQuery( q );
    rb.setSortSpec( parser.getSort(true) );
    rb.setQparser(parser);
    
    final String cursorStr = rb.req.getParams().get(CursorMarkParams.CURSOR_MARK_PARAM);
    if (null != cursorStr) {
      final CursorMark cursorMark = new CursorMark(rb.req.getSchema(),
                                                   rb.getSortSpec());
      cursorMark.parseSerializedTotem(cursorStr);
      rb.setCursorMark(cursorMark);
    }

    String[] fqs = req.getParams().getParams(CommonParams.FQ);
    if (fqs!=null && fqs.length!=0) {
      List<Query> filters = rb.getFilters();
      // if filters already exists, make a copy instead of modifying the original
      filters = filters == null ? new ArrayList<Query>(fqs.length) : new ArrayList<Query>(filters);
      for (String fq : fqs) {
        if (fq != null && fq.trim().length()!=0) {
          QParser fqp = QParser.getParser(fq, null, req);
          filters.add(fqp.getQuery());
        }
      }
      // only set the filters if they are not empty otherwise
      // fq=&someotherParam= will trigger all docs filter for every request 
      // if filter cache is disabled
      if (!filters.isEmpty()) {
        rb.setFilters( filters );
      }
    }
  } catch (SyntaxError e) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
  }

  if (params.getBool(GroupParams.GROUP, false)) {
    prepareGrouping(rb);
  }
}