本文整理汇总了Java中org.apache.nutch.indexer.IndexingException类的典型用法代码示例。如果您正苦于以下问题:Java IndexingException类的具体用法?Java IndexingException怎么用?Java IndexingException使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
IndexingException类属于org.apache.nutch.indexer包,在下文中一共展示了IndexingException类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
* This will take the metatags that you have listed in your "urlmeta.tags"
* property, and looks for them inside the CrawlDatum object. If they exist,
* this will add it as an attribute inside the NutchDocument.
*
* @see IndexingFilter#filter
*/
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
if (conf != null)
this.setConf(conf);
if (urlMetaTags == null || doc == null)
return doc;
for (String metatag : urlMetaTags) {
Text metadata = (Text) datum.getMetaData().get(new Text(metatag));
if (metadata != null)
doc.add(metatag, metadata.toString());
}
return doc;
}
示例2: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
// check if LANGUAGE found, possibly put there by HTMLLanguageParser
String lang = parse.getData().getParseMeta().get(Metadata.LANGUAGE);
// check if HTTP-header tels us the language
if (lang == null) {
lang = parse.getData().getContentMeta().get(Response.CONTENT_LANGUAGE);
}
if (lang == null || lang.length() == 0) {
lang = "unknown";
}
doc.add("lang", lang);
return doc;
}
示例3: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
* {@inheritDoc}
*/
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
if (doc != null) {
if (FIELDREPLACERS_BY_HOST.size() > 0) {
this.doReplace(doc, "host", FIELDREPLACERS_BY_HOST);
}
if (FIELDREPLACERS_BY_URL.size() > 0) {
this.doReplace(doc, "url", FIELDREPLACERS_BY_URL);
}
}
return doc;
}
示例4: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
throws IndexingException {
ParseData dataP = parse.getData();
Metadata meta = dataP.getParseMeta();
boolean index = false;
for (String key : meta.names()) {
if(key.equals("ogc_service"))
index = true;
String value = meta.get(key);
LOG.info("Adding " + url + " to NutchDocument");
doc.add(key, value);
}
/* Return the document if it is an ogc service, otherwise return null */
return index ? doc : null;
}
示例5: testOgcIndexingFilter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Test
public void testOgcIndexingFilter() throws FileNotFoundException, URISyntaxException, IndexingException {
File f = new File(getClass().getResource("testWMS.xml").toURI());
@SuppressWarnings("resource")
String contentValue = new Scanner(f).useDelimiter("\\Z").next();
ParseResult testParseResult = Utils.createParseResultWithMetadata(new Metadata(), url);
Content testContent = Utils.createContent(url, contentValue);
OgcIndexingFilter indexingFilter = new OgcIndexingFilter();
OgcParseFilter parseFilter = new OgcParseFilter();
ParseResult res = parseFilter.filter(testContent, testParseResult, null, null);
parse = res.get(url);
NutchDocument doc = indexingFilter.filter(nutchDocument, parse, urlText, datum, inlinks);
assertTrue("Comprobación de que el campo ogc_version esta indexado",
doc.getFieldNames().contains("ogc_version"));
assertTrue("Comprobación de que el campo ogc_service esta indexado",
doc.getFieldNames().contains("ogc_service"));
assertTrue("Comprobación de que el campo raw_content esta indexado",
doc.getFieldNames().contains("raw_content"));
}
示例6: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
* This will take the metatags that you have listed in your "urlmeta.tags"
* property, and looks for them inside the CrawlDatum object. If they exist,
* this will add it as an attribute inside the NutchDocument.
*
* @see IndexingFilter#filter
*/
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
if (conf != null)
this.setConf(conf);
if (urlMetaTags == null || doc == null)
return doc;
for (String metatag : urlMetaTags) {
Text metadata = (Text) datum.getMetaData().get(new Text(metatag));
if (metadata != null)
doc.add(metatag, metadata.toString());
}
return doc;
}
示例7: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
throws IndexingException {
// check if LANGUAGE found, possibly put there by HTMLLanguageParser
String lang = parse.getData().getParseMeta().get(Metadata.LANGUAGE);
// check if HTTP-header tels us the language
if (lang == null) {
lang = parse.getData().getContentMeta().get(Response.CONTENT_LANGUAGE);
}
if (lang == null || lang.length() == 0) {
lang = "unknown";
}
doc.add("lang", lang);
return doc;
}
示例8: getSiteHashFromJsonStream
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
* gets the siteHash from the received Json data in form of InputStream
*
* @param stream
* with Json data
* @return siteHash
* @throws IndexingException
*/
protected String getSiteHashFromJsonStream(InputStream stream)
throws IndexingException {
try {
JsonNode rootNode = jsonMapper.readValue(stream, JsonNode.class);
String siteHash = rootNode.get("sitehash").getTextValue();
LOG.info("TYPO3 Solr siteHash retrieved: " + siteHash);
return siteHash;
} catch (Exception e) {
LOG.error("ERROR! could not receive correct siteHash data from the Solr TYPO3 Api");
throw (new IndexingException(e));
} finally {
if (stream != null) {
try {
stream.close();
} catch (IOException streamException) {
LOG.error(streamException.getMessage());
}
}
}
}
示例9: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
// convert ISO date to time stamp
String isoDate = conf.get(CONF_ENDTIME_PROPERTY, "1970-01-01T00:00:00Z");
long epoch = 0;
try {
epoch = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").parse(isoDate).getTime();
} catch (ParseException e) {
LOG.error("ERROR! Cannot parse date, must fit pattern yyyy-MM-dd'T'HH:mm:ssZ : " + isoDate);
}
// Index the endtime
doc.add(INDEXING_FIELD, new Date(epoch));
return doc;
}
示例10: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument document, String s, WebPage webPage) throws IndexingException {
if (storageField != null) {
CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
try {
String strippedContent = decoder.decode(webPage.getMetadata().get(new Utf8(storageField))).toString();
if (strippedContent != null) {
document.add(storageField, strippedContent);
}
} catch (CharacterCodingException e) {
e.printStackTrace();
}
}
return document;
}
示例11: testOgcIndexingFilter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Test
public void testOgcIndexingFilter() throws FileNotFoundException, URISyntaxException, IndexingException {
int results = th.execQuery("agua");
assertEquals(results, 1);
results = th.execQuery("вода");
assertEquals(results, 1);
results = th.execQuery("Mar");
assertEquals(results, 30);
}
示例12: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
// Check if some Rel-Tags found, possibly put there by RelTagParser
String[] tags = parse.getData().getParseMeta()
.getValues(RelTagParser.REL_TAG);
if (tags != null) {
for (int i = 0; i < tags.length; i++) {
doc.add("tag", tags[i]);
}
}
return doc;
}
示例13: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text urlText,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
try {
URL url = new URL(urlText.toString());
DomainSuffix d = URLUtil.getDomainSuffix(url);
doc.add("tld", d.getDomain());
} catch (Exception ex) {
LOG.warn(ex.toString());
}
return doc;
}
示例14: filter
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
String url_s = url.toString();
addTime(doc, parse.getData(), url_s, datum);
addLength(doc, parse.getData(), url_s);
addType(doc, parse.getData(), url_s, datum);
resetTitle(doc, parse.getData(), url_s);
return doc;
}
示例15: assertContentType
import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
private void assertContentType(Configuration conf, String source,
String expected) throws IndexingException {
Metadata metadata = new Metadata();
metadata.add(Response.CONTENT_TYPE, source);
MoreIndexingFilter filter = new MoreIndexingFilter();
filter.setConf(conf);
NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl(
"text", new ParseData(new ParseStatus(), "title", new Outlink[0],
metadata)), new Text("http://www.example.com/"), new CrawlDatum(),
new Inlinks());
Assert.assertEquals("mime type not detected", expected,
doc.getFieldValue("type"));
}