Java AbstractSiteMap类代码示例

import crawlercommons.sitemaps.AbstractSiteMap; //导入依赖的package包/类
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
    String localName = super.currentElement();
    String value = String.valueOf(ch, start, length);
    if ("pubDate".equals(localName)) {
        lastMod = AbstractSiteMap.normalizeRSSTimestamp(value);
        if ("channel".equals(super.currentElementParent())) {
            sitemap.setLastModified(lastMod);
        }
    } else if ("link".equals(localName)) {
        String href = value;
        LOG.debug("href = {}", href);
        try {
            loc = new URL(href);
            valid = urlIsValid(sitemap.getBaseUrl(), href);
        } catch (MalformedURLException e) {
            LOG.trace("Can't create an entry with a bad URL", e);
            LOG.debug("Bad url: [{}]", href);
        }
    }
}
 

import crawlercommons.sitemaps.AbstractSiteMap; //导入依赖的package包/类
public List<AbstractSiteMap> getSitemapsForUrl(String sitemapUrl) {
	List<AbstractSiteMap> sitemaps = new ArrayList<>();

	SiteMapParser siteMapParser = new SiteMapParser();
	try {
		Uri uri = Uri.create(sitemapUrl);
		Blob blob = Requesters.of(uri.getScheme()).get().get(uri);
		String contentType = blob.getMetadata().getContentMetadata().contentType() != null ? blob.getMetadata().getContentMetadata().contentType()
				: "text/xml";

		AbstractSiteMap sitemap = siteMapParser.parseSiteMap(contentType, IOUtils.toByteArray(blob.getPayload().openStream()), new URL(sitemapUrl));

		if (sitemap.isIndex()) {
			sitemaps.addAll(((SiteMapIndex) sitemap).getSitemaps());
		} else {
			sitemaps.add(sitemap);
		}
	} catch (Exception e) {
		log.debug("", e);
	}
	return sitemaps;
}
 

import crawlercommons.sitemaps.AbstractSiteMap; //导入依赖的package包/类
public AbstractSiteMap getSiteMap() {
    return sitemap;
}
 

import crawlercommons.sitemaps.AbstractSiteMap; //导入依赖的package包/类
public AbstractSiteMap getSiteMap() {
    if (delegate == null)
        return null;
    return delegate.getSiteMap();
}
 

import crawlercommons.sitemaps.AbstractSiteMap; //导入依赖的package包/类
protected Analysis buildReport(Job job, Blob blob) {
	Analysis report;
	if (blob.getMetadata().getUri().getScheme().startsWith("http")) {
		HttpAnalysis temp = new HttpAnalysis();

		// Robots.txt
		Uri pageURL = blob.getMetadata().getUri();
		String robotsTxtUrl = pageURL.getScheme() + "://" + pageURL.getHost() + ":" + pageURL.getPort() + "/robots.txt";
		ExtendedRobotRules robotRules = RobotsTxtUtils.getRobotRules(robotsTxtUrl);
		temp.robotRules(robotRules);

		// Sitemaps
		if (robotRules != null && robotRules.getSitemaps() != null) {
			Map<String, List<AbstractSiteMap>> sitemaps = new HashMap<>();
			robotRules.getSitemaps().forEach(url -> {
				List<AbstractSiteMap> results = getSitemapsForUrl(url);
				sitemaps.put(url, results);
			});
			temp.sitemaps(sitemaps);
		}
		report = temp;
	} else {
		report = new Analysis();
	}

	if (job.getDefinition().getExtractors() != null) {
		Map<String, Instance<?>> cachedSelectors = new HashMap<>();

		// Page extraction
		if (job.getDefinition().getExtractors().getData() != null) {
			Map<String, List<Document>> documentsByExtractor = job.getDefinition().getExtractors().getData().stream()
					.map(ex -> Pair.of(ex.getName(), extractorService.extractThenFormat(cachedSelectors, blob, ex)))
					.filter(pair -> pair != null && pair.getKey() != null && pair.getValue() != null)
					.collect(Collectors.toMap(key -> key.getLeft(), value -> value.getRight()));
			report.documents(documentsByExtractor);
		}

		// Link extraction
		if (job.getDefinition().getExtractors().getOutlinks() != null) {
			Map<String, Pair<Set<Link>, Set<Link>>> outlinksByExtractor = job.getDefinition().getExtractors().getOutlinks().stream().map(ol -> {
				return Pair.of(ol.getName(), extractorService.extractAndFilterOutlinks(job, blob.getMetadata().getUri(), cachedSelectors, blob, ol));
			}).collect(Collectors.toMap(key -> key.getLeft(), value -> value.getRight()));

			report.outlinks(Maps.transformEntries(outlinksByExtractor, (key, entries) -> entries.getLeft()));
			report.filteredOutlinks(Maps.transformEntries(outlinksByExtractor, (key, entries) -> entries.getRight()));
		}

	}

	report.metadata(blob.getMetadata());
	return report;
}