本文整理匯總了Java中com.digitalpebble.stormcrawler.parse.ParseData.setMetadata方法的典型用法代碼示例。如果您正苦於以下問題：Java ParseData.setMetadata方法的具體用法？Java ParseData.setMetadata怎麽用？Java ParseData.setMetadata使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類com.digitalpebble.stormcrawler.parse.ParseData的用法示例。
在下文中一共展示了ParseData.setMetadata方法的2個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。
示例1: execute
import com.digitalpebble.stormcrawler.parse.ParseData; //導入方法依賴的package包/類
@Override
public void execute(Tuple tuple) {
Metadata metadata = (Metadata) tuple.getValueByField("metadata");
byte[] content = tuple.getBinaryByField("content");
String url = tuple.getStringByField("url");
boolean isFeed = Boolean.valueOf(metadata.getFirstValue(isFeedKey));
if (!isFeed) {
String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE);
if (ct != null) {
for (String clue : mimeTypeClues) {
if (ct.contains(clue)) {
isFeed = true;
metadata.setValue(isFeedKey, "true");
LOG.info("Feed detected from content type <{}> for {}",
ct, url);
break;
}
}
}
}
if (!isFeed) {
if (contentDetector.matches(content)) {
isFeed = true;
metadata.setValue(isFeedKey, "true");
LOG.info("Feed detected from content: {}", url);
}
}
if (isFeed) {
// do not parse but run parse filters
ParseResult parse = new ParseResult();
ParseData parseData = parse.get(url);
parseData.setMetadata(metadata);
parseFilters.filter(url, content, null, parse);
// emit status
collector.emit(Constants.StatusStreamName, tuple,
new Values(url, metadata, Status.FETCHED));
} else {
// pass on
collector.emit(tuple, tuple.getValues());
}
collector.ack(tuple);
}
示例2: execute
import com.digitalpebble.stormcrawler.parse.ParseData; //導入方法依賴的package包/類
@Override
public void execute(Tuple tuple) {
Metadata metadata = (Metadata) tuple.getValueByField("metadata");
byte[] content = tuple.getBinaryByField("content");
String url = tuple.getStringByField("url");
boolean isSitemap = Boolean.valueOf(
metadata.getFirstValue(SiteMapParserBolt.isSitemapKey));
boolean isNewsSitemap = Boolean.valueOf(
metadata.getFirstValue(NewsSiteMapParserBolt.isSitemapNewsKey));
if (!isNewsSitemap || !isSitemap) {
int match = contentDetector.getFirstMatch(content);
if (match >= 0) {
// a sitemap, not necessarily a news sitemap
isSitemap = true;
metadata.setValue(SiteMapParserBolt.isSitemapKey, "true");
if (match <= NewsSiteMapParserBolt.contentCluesSitemapNewsMatchUpTo) {
isNewsSitemap = true;
LOG.info("{} detected as news sitemap based on content",
url);
metadata.setValue(NewsSiteMapParserBolt.isSitemapNewsKey,
"true");
}
}
}
if (isSitemap) {
// do not parse but run parse filters
ParseResult parse = new ParseResult();
ParseData parseData = parse.get(url);
parseData.setMetadata(metadata);
parseFilters.filter(url, content, null, parse);
// emit status
collector.emit(Constants.StatusStreamName, tuple,
new Values(url, metadata, Status.FETCHED));
} else {
// pass on
collector.emit(tuple, tuple.getValues());
}
collector.ack(tuple);
}