Java IndexingException类代码示例

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
 * This will take the metatags that you have listed in your "urlmeta.tags"
 * property, and looks for them inside the CrawlDatum object. If they exist,
 * this will add it as an attribute inside the NutchDocument.
 * 
 * @see IndexingFilter#filter
 */
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {
  if (conf != null)
    this.setConf(conf);

  if (urlMetaTags == null || doc == null)
    return doc;

  for (String metatag : urlMetaTags) {
    Text metadata = (Text) datum.getMetaData().get(new Text(metatag));

    if (metadata != null)
      doc.add(metatag, metadata.toString());
  }

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {

  // check if LANGUAGE found, possibly put there by HTMLLanguageParser
  String lang = parse.getData().getParseMeta().get(Metadata.LANGUAGE);

  // check if HTTP-header tels us the language
  if (lang == null) {
    lang = parse.getData().getContentMeta().get(Response.CONTENT_LANGUAGE);
  }

  if (lang == null || lang.length() == 0) {
    lang = "unknown";
  }

  doc.add("lang", lang);

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {

  if (doc != null) {
    if (FIELDREPLACERS_BY_HOST.size() > 0) {
      this.doReplace(doc, "host", FIELDREPLACERS_BY_HOST);
    }

    if (FIELDREPLACERS_BY_URL.size() > 0) {
      this.doReplace(doc, "url", FIELDREPLACERS_BY_URL);
    }
  }

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
		throws IndexingException {
	ParseData dataP = parse.getData();
	Metadata meta = dataP.getParseMeta();
	boolean index = false;
	
	for (String key : meta.names()) {
		if(key.equals("ogc_service"))
			index = true;
		String value = meta.get(key);
		LOG.info("Adding " + url + " to NutchDocument");
		doc.add(key, value);
	}
	/* Return the document if it is an ogc service, otherwise return null */
	return index ? doc : null;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Test
public void testOgcIndexingFilter() throws FileNotFoundException, URISyntaxException, IndexingException {
	File f = new File(getClass().getResource("testWMS.xml").toURI());
	@SuppressWarnings("resource")
	String contentValue = new Scanner(f).useDelimiter("\\Z").next();
	ParseResult testParseResult = Utils.createParseResultWithMetadata(new Metadata(), url);
	Content testContent = Utils.createContent(url, contentValue);

	OgcIndexingFilter indexingFilter = new OgcIndexingFilter();
	OgcParseFilter parseFilter = new OgcParseFilter();

	ParseResult res = parseFilter.filter(testContent, testParseResult, null, null);
	parse = res.get(url);

	NutchDocument doc = indexingFilter.filter(nutchDocument, parse, urlText, datum, inlinks);

	assertTrue("Comprobación de que el campo ogc_version esta indexado",
			doc.getFieldNames().contains("ogc_version"));
	assertTrue("Comprobación de que el campo ogc_service esta indexado",
			doc.getFieldNames().contains("ogc_service"));
	assertTrue("Comprobación de que el campo raw_content esta indexado",
			doc.getFieldNames().contains("raw_content"));
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
 * This will take the metatags that you have listed in your "urlmeta.tags"
 * property, and looks for them inside the CrawlDatum object. If they exist,
 * this will add it as an attribute inside the NutchDocument.
 * 
 * @see IndexingFilter#filter
 */
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
		CrawlDatum datum, Inlinks inlinks) throws IndexingException {
	if (conf != null)
		this.setConf(conf);

	if (urlMetaTags == null || doc == null)
		return doc;

	for (String metatag : urlMetaTags) {
		Text metadata = (Text) datum.getMetaData().get(new Text(metatag));

		if (metadata != null)
			doc.add(metatag, metadata.toString());
	}

	return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
  throws IndexingException {

  // check if LANGUAGE found, possibly put there by HTMLLanguageParser
  String lang = parse.getData().getParseMeta().get(Metadata.LANGUAGE);

  // check if HTTP-header tels us the language
  if (lang == null) {
      lang = parse.getData().getContentMeta().get(Response.CONTENT_LANGUAGE);
  }

  if (lang == null || lang.length() == 0) {
    lang = "unknown";
  }

  doc.add("lang", lang);

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
/**
 * gets the siteHash from the received Json data in form of InputStream
 *
 * @param stream
 *            with Json data
 * @return siteHash
 * @throws IndexingException
 */
protected String getSiteHashFromJsonStream(InputStream stream)
		throws IndexingException {
	try {
		JsonNode rootNode = jsonMapper.readValue(stream, JsonNode.class);
		String siteHash = rootNode.get("sitehash").getTextValue();

		LOG.info("TYPO3 Solr siteHash retrieved: " + siteHash);

		return siteHash;
	} catch (Exception e) {
		LOG.error("ERROR! could not receive correct siteHash data from the Solr TYPO3 Api");

		throw (new IndexingException(e));
	} finally {
		if (stream != null) {
			try {
				stream.close();
			} catch (IOException streamException) {
				LOG.error(streamException.getMessage());
			}
		}
	}
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
		CrawlDatum datum, Inlinks inlinks) throws IndexingException {

	// convert ISO date to time stamp
	String isoDate = conf.get(CONF_ENDTIME_PROPERTY, "1970-01-01T00:00:00Z");
	long epoch = 0;
	try {
		epoch = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").parse(isoDate).getTime();
	} catch (ParseException e) {
		LOG.error("ERROR! Cannot parse date, must fit pattern yyyy-MM-dd'T'HH:mm:ssZ : " + isoDate);
	}

	// Index the endtime
	doc.add(INDEXING_FIELD, new Date(epoch));

	return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Override
public NutchDocument filter(NutchDocument document, String s, WebPage webPage) throws IndexingException {
    if (storageField != null) {
        CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
        try {
            String strippedContent = decoder.decode(webPage.getMetadata().get(new Utf8(storageField))).toString();
            if (strippedContent != null) {
                document.add(storageField, strippedContent);
            }
        } catch (CharacterCodingException e) {
            e.printStackTrace();
        }
    }

    return document;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
@Test
public void testOgcIndexingFilter() throws FileNotFoundException, URISyntaxException, IndexingException {
	int results = th.execQuery("agua");
	assertEquals(results, 1);
	results = th.execQuery("вода");
	assertEquals(results, 1);
	results = th.execQuery("Mar");
	assertEquals(results, 30);

}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {

  // Check if some Rel-Tags found, possibly put there by RelTagParser
  String[] tags = parse.getData().getParseMeta()
      .getValues(RelTagParser.REL_TAG);
  if (tags != null) {
    for (int i = 0; i < tags.length; i++) {
      doc.add("tag", tags[i]);
    }
  }

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text urlText,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {

  try {
    URL url = new URL(urlText.toString());
    DomainSuffix d = URLUtil.getDomainSuffix(url);

    doc.add("tld", d.getDomain());

  } catch (Exception ex) {
    LOG.warn(ex.toString());
  }

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
    CrawlDatum datum, Inlinks inlinks) throws IndexingException {

  String url_s = url.toString();

  addTime(doc, parse.getData(), url_s, datum);
  addLength(doc, parse.getData(), url_s);
  addType(doc, parse.getData(), url_s, datum);
  resetTitle(doc, parse.getData(), url_s);

  return doc;
}
 

import org.apache.nutch.indexer.IndexingException; //导入依赖的package包/类
private void assertContentType(Configuration conf, String source,
    String expected) throws IndexingException {
  Metadata metadata = new Metadata();
  metadata.add(Response.CONTENT_TYPE, source);
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);
  NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl(
      "text", new ParseData(new ParseStatus(), "title", new Outlink[0],
          metadata)), new Text("http://www.example.com/"), new CrawlDatum(),
      new Inlinks());
  Assert.assertEquals("mime type not detected", expected,
      doc.getFieldValue("type"));
}