Java DocIdPusher.pushDocIds方法代码示例

import com.google.enterprise.adaptor.DocIdPusher; //导入方法依赖的package包/类
private void getDocIdsSiteCollectionOnly(DocIdPusher pusher)
    throws InterruptedException,IOException {
  log.entering("SharePointAdaptor", "getDocIdsSiteCollectionOnly", pusher);
  SiteAdaptor scAdaptor = getSiteAdaptor(sharePointUrl.getSharePointUrl(), 
      sharePointUrl.getSharePointUrl());
  SiteDataClient scClient = scAdaptor.getSiteDataClient();
  Site site = scClient.getContentSite();
  String siteCollectionUrl = getCanonicalUrl(site.getMetadata().getURL());
  // Reset site collection URL instance to use correct URL.
  scAdaptor = getSiteAdaptor(siteCollectionUrl, siteCollectionUrl);
  DocId siteCollectionDocId = scAdaptor.encodeDocId(siteCollectionUrl);
  pusher.pushDocIds(Arrays.asList(siteCollectionDocId));
  Map<GroupPrincipal, Collection<Principal>> groupDefs 
      = new HashMap<GroupPrincipal, Collection<Principal>>();
  groupDefs.putAll(scAdaptor.computeMembersForGroups(site.getGroups()));
  String siteId = site.getMetadata().getID();
  sitePushGroupDefinitions(siteId, pusher, groupDefs);
  log.exiting("SharePointAdaptor", "getDocIdsSiteCollectionOnly");
}
 

import com.google.enterprise.adaptor.DocIdPusher; //导入方法依赖的package包/类
/** Crawls/pushes ids of all people from all LDAP Servers. */
@Override
public void getDocIds(DocIdPusher pusher) throws InterruptedException,
    IOException {
  log.entering("LdapAdaptor", "getDocIds");
  String badHosts = ""; // built up as we run into errors
  NamingException lastException = null;
  for (int serverNumber = 0; serverNumber < servers.size(); serverNumber++) {
    LdapServer server = servers.get(serverNumber);
    try {
      server.ensureConnectionIsCurrent();
      Set<LdapPerson> entities = server.scanAll();
      ArrayList<DocId> docIds = new ArrayList<DocId>();
      log.fine("received " + entities.size() + " entities from server");
      for (LdapPerson person : entities) {
        docIds.add(makeDocId(serverNumber, person.getDn()));
      }
      log.fine("About to push " + docIds.size() + " docIds for host "
          + server.getHostName() + "");
      pusher.pushDocIds(docIds);
      log.finer("Done with push of " + docIds.size() + " docIds for host "
          + server.getHostName() + "");
    } catch (NamingException ne) {
      String host = server.getHostName();
      badHosts += "," + host;
      lastException = ne;
      log.log(Level.WARNING, "Could not get entitites from " + host + "", ne);
      continue; // just log for now; throw exception at very end.
    }
  }
  log.exiting("LdapAdaptor", "getDocIds", (lastException != null));
  if (lastException != null) {
    throw new IOException("Could not get entities from the following "
        + "server(s): " + badHosts.substring(1) + "", lastException);
  }
}
 

import com.google.enterprise.adaptor.DocIdPusher; //导入方法依赖的package包/类
@Override
public void getDocIds(DocIdPusher pusher) throws InterruptedException {
  Authentication authentication = this.soapFactory.newAuthentication();
  String authenticationToken =
      getAuthenticationToken(this.username, this.password);
  DocumentManagement documentManagement =
      soapFactory.newDocumentManagement(authenticationToken);
  ArrayList<DocId> docIds = new ArrayList<DocId>();
  for (StartPoint startPoint : this.startPoints) {
    if (isValidStartPoint(startPoint, documentManagement)) {
      docIds.add(
          new DocId(startPoint.getName() + ":" + startPoint.getNodeId()));
    }
  }
  log.log(Level.FINER, "Sending doc ids: {0}", docIds);
  pusher.pushDocIds(docIds);

  if (!this.markAllDocsAsPublic) {
    MemberService memberService =
        this.soapFactory.newMemberService(documentManagement);
    SOAPFaultException groupsException = null;
    Groups groups = new Groups(memberService);
    try {
      groups.addGroups();
      if (this.publicAccessGroupEnabled) {
        groups.addPublicAccessGroup("[Public Access]", this.localNamespace);
      }
    } catch (SOAPFaultException e) {
      groupsException = e;
      log.log(Level.WARNING, "Error while reading groups", e);
    }

    // If some groups were found, push them; otherwise, if an
    // exception was thrown, rethrow it as before.
    Map<GroupPrincipal, List<Principal>> groupDefinitions =
        groups.getGroupDefinitions();
    if (groupDefinitions.size() > 0) {
      DocIdPusher.FeedType feedType = DocIdPusher.FeedType.REPLACE;
      if (groups.isPartialResults() || groupsException != null) {
        feedType = DocIdPusher.FeedType.INCREMENTAL;
        log.log(Level.FINE, "Sending partial groups results");
      }
      pusher.pushGroupDefinitions(groupDefinitions,
          /* case sensitive */ true, feedType,
          /* group source */ null, /* exception handler */ null);
    } else if (groupsException != null) {
      throw groupsException;
    }
  }
}
 

import com.google.enterprise.adaptor.DocIdPusher; //导入方法依赖的package包/类
@Override
public void getModifiedDocIds(DocIdPusher pusher)
    throws IOException, InterruptedException {
  // Log in using CWS and use that token for the search.
  Authentication authentication = this.soapFactory.newAuthentication();
  String authenticationToken =
      getAuthenticationToken(this.username, this.password);

  ArrayList<DocId> docIds = new ArrayList<>();
  int resultCount = 0;
  do {
    String query = getLastModifiedQuery();
    log.log(Level.FINER, "getModifiedDocIds query: {0}", query);
    Element documentElement =
        getXmlSearchResults(authenticationToken, query);
    resultCount = getXmlSearchCount(documentElement);
    log.log(Level.FINER, "getModifiedDocIds result count: " + resultCount);
    if (resultCount > 0) {
      NodeList searchResults =
          documentElement.getElementsByTagName("SearchResult");
      for (int i = 0; i < searchResults.getLength(); i++) {
        String docId = getXmlSearchDocId((Element) searchResults.item(i));
        if (docId != null) {
          docIds.add(new DocId(docId));
        }
      }
      // Cache the last object's id/date/time
      if (docIds.size() > 0) {
        Long nodeId = Longs.tryParse(
            docIds.get(docIds.size() - 1).getUniqueId().split(":")[1]);
        DocumentManagement documentManagement =
            this.soapFactory.newDocumentManagement(authenticationToken);
        Node node = getNodeById(documentManagement, nodeId);
        if (node != null) {
          XMLGregorianCalendar xmlCalendar = node.getModifyDate();
          if (xmlCalendar != null) {
            Date date = xmlCalendar.toGregorianCalendar().getTime();
            this.lastModDocId = nodeId;
            this.lastModDate = new SimpleDateFormat("yyyyMMdd").format(date);
            this.lastModTime = new SimpleDateFormat("HHmmss").format(date);
          }
        }
      }
    }
  } while (resultCount > 0);
  log.log(Level.FINER, "Sending modified doc ids: {0}", docIds);
  pusher.pushDocIds(docIds);
}
 

import com.google.enterprise.adaptor.DocIdPusher; //导入方法依赖的package包/类
public void getDocIds(DocIdPusher pusher)
    throws RemoteException, InterruptedException {
  int index = -1;
  List<DocId> profilesToPush = new ArrayList<DocId>();
  do {
    GetUserProfileByIndexResult nextProfile;
    try {
      nextProfile = userProfileServiceWS.getUserProfileByIndex(index);
    } catch (WebServiceException e) {
      log.log(Level.WARNING,
          "Error fetching user profile at index {0}", index);
      log.log(Level.WARNING,
          "Exception for getUserProfileByIndex : ", e);
      if (isDueToXmlParseException(e)) {
        log.log(Level.WARNING,
          "Skipping user profile at index {0} due to "
              + "XML parsing error", index);            
        index = index + 1;
        continue;
      }
      // Flushing available docids
      pusher.pushDocIds(profilesToPush);
      throw e;
    }
    if (nextProfile == null) {
      break;
    }
    index = Integer.parseInt(nextProfile.getNextValue());
    log.log(Level.FINEST, "Next Index is {0}", index);
    ArrayOfPropertyData profileProperties = nextProfile.getUserProfile();
    String userAccountName = getUserProfilePropertySingleValue(
        profileProperties, PROFILE_ACCOUNTNAME_PROPERTY);
    if (!Strings.isNullOrEmpty(userAccountName)) {
      profilesToPush.add(new DocId(SOCIAL_ID_PREFIX + userAccountName));
      log.log(Level.FINEST, "Adding Doc ID {0}",
          SOCIAL_ID_PREFIX + userAccountName);
    }
    if (profilesToPush.size() == 500) {
      pusher.pushDocIds(profilesToPush);
      profilesToPush.clear();
    }
  } while (index != -1); // For last profile next value will be -1
  pusher.pushDocIds(profilesToPush);
}