Java WARCRecordInfo.setContentLength方法代码示例

import org.archive.io.warc.WARCRecordInfo; //导入方法依赖的package包/类
protected URI writeRequest(URI id) throws IOException, ParseException {
  WARCRecordInfo record = new WARCRecordInfo();

  record.setType(WARCConstants.WARCRecordType.request);
  record.setUrl(getUrl());
  record.setCreate14DigitDate(DateUtils
      .getLog14Date(Long.parseLong(metadata.get("nutch.fetch.time"))));
  record.setMimetype(WARCConstants.HTTP_REQUEST_MIMETYPE);
  record.setRecordId(GENERATOR.getRecordID());

  if (id != null) {
    ANVLRecord headers = new ANVLRecord();
    headers.addLabelValue(WARCConstants.HEADER_KEY_CONCURRENT_TO,
        '<' + id.toString() + '>');
    record.setExtraHeaders(headers);
  }

  ByteArrayOutputStream output = new ByteArrayOutputStream();

  output.write(metadata.get("_request_").getBytes());
  record.setContentLength(output.size());
  record.setContentStream(new ByteArrayInputStream(output.toByteArray()));

  writer.writeRecord(record);

  return record.getRecordId();
}
 

import org.archive.io.warc.WARCRecordInfo; //导入方法依赖的package包/类
public WARCRecordInfo getWarcRecordInfo(Page page, URI uri) throws IOException {
    WARCRecordInfo warcRecord = new WARCRecordInfo();

    warcRecord.setUrl(page.getFinalUrl());
    warcRecord.setRecordId(uri);
    warcRecord.setType(WARCRecordType.response);
    warcRecord.setMimetype(WARCConstants.HTTP_RESPONSE_MIMETYPE);

    // Store fetch times using ISO-8601 format
    Date fetchTime = createFetchTimeDate(page);
    warcRecord.setCreate14DigitDate(dateFormater.get().format(fetchTime));

    // Re-create response body based on content bytes and response headers
    byte[] contentBytes = createContentBytes(page);
    warcRecord.setContentLength(contentBytes.length);
    warcRecord.setContentStream(new ByteArrayInputStream(contentBytes));

    // Store ACHE-specific metadata as non-standard extension header fields
    if (page.getTargetRelevance() != null) {
        TargetRelevance targetRelevance = page.getTargetRelevance();
        warcRecord.addExtraHeader("ACHE-IsRelevant", String.valueOf(targetRelevance.isRelevant()));
        warcRecord.addExtraHeader("ACHE-Relevance", String.format("%.10f", targetRelevance.getRelevance()));
    }
    warcRecord.addExtraHeader("ACHE-Requested-URL", page.getRequestedUrl());

    return warcRecord;
}
 

import org.archive.io.warc.WARCRecordInfo; //导入方法依赖的package包/类
protected URI writeResponse() throws IOException, ParseException {
  WARCRecordInfo record = new WARCRecordInfo();

  record.setType(WARCConstants.WARCRecordType.response);
  record.setUrl(getUrl());

  String fetchTime;

  record.setCreate14DigitDate(DateUtils
      .getLog14Date(Long.parseLong(metadata.get("nutch.fetch.time"))));
  record.setMimetype(WARCConstants.HTTP_RESPONSE_MIMETYPE);
  record.setRecordId(GENERATOR.getRecordID());

  String IP = getResponseAddress();

  if (StringUtils.isNotBlank(IP))
    record.addExtraHeader(WARCConstants.HEADER_KEY_IP, IP);

  if (ParseSegment.isTruncated(content))
    record.addExtraHeader(WARCConstants.HEADER_KEY_TRUNCATED, "unspecified");

  ByteArrayOutputStream output = new ByteArrayOutputStream();

  String httpHeaders = metadata.get("_response.headers_");

  if (StringUtils.isNotBlank(httpHeaders)) {
    output.write(httpHeaders.getBytes());
  } else {
    // change the record type to resource as we not have information about
    // the headers
    record.setType(WARCConstants.WARCRecordType.resource);
    record.setMimetype(content.getContentType());
  }

  output.write(getResponseContent().getBytes());

  record.setContentLength(output.size());
  record.setContentStream(new ByteArrayInputStream(output.toByteArray()));

  if (output.size() > 0) {
    // avoid generating a 0 sized record, as the webarchive library will
    // complain about it
    writer.writeRecord(record);
  }

  return record.getRecordId();
}
 

import org.archive.io.warc.WARCRecordInfo; //导入方法依赖的package包/类
protected void write(final WARCWriter writer, final Content content)
        throws IOException, ParseException
{
    WARCRecordInfo recordInfo = new WARCRecordInfo();
    recordInfo.setUrl(content.getUrl());

    byte[] byteContent = content.getContent();

    // skip empty records
    if (byteContent.length == 0) {
        return;
    }

    recordInfo.setContentStream(new ByteArrayInputStream(byteContent));
    recordInfo.setContentLength(byteContent.length);
    recordInfo.setEnforceLength(true);

    String warcDateString = DEFAULT_WARC_DATE;

    // convert date to WARC-Date format
    String date = content.getMetadata().get("Date");
    if (date != null) {
        try {
            warcDateString = String.valueOf(
                    new SimpleDateFormat("EEE, dd MMM yyyy kk:mm:ss ZZZ", Locale.ENGLISH)
                            .parse(date).getTime());
        }
        catch (ParseException ex) {
            // ignore
        }
    }

    recordInfo.setCreate14DigitDate(warcDateString);

    recordInfo.setType(WARCConstants.WARCRecordType.response);
    recordInfo.setMimetype(WARCConstants.HTTP_RESPONSE_MIMETYPE);
    recordInfo.setRecordId(generator.getRecordID());

    // add some extra headers from nutch
    Set<String> extraHeaders = new HashSet<String>(
            Arrays.asList("nutch.crawl.score", "nutch.segment.name", "Set-Cookie",
                    "Content-Type", "Server", "Pragma", "Cache-Control"));

    for (String extraHeader : extraHeaders) {
        String value = content.getMetadata().get(extraHeader);
        if (value != null) {
            recordInfo.addExtraHeader("Nutch_" + extraHeader, value);
        }
    }

    // apply filters
    boolean acceptExport = true;
    for (ExportContentFilter filter : filters) {
        acceptExport &= filter.acceptContent(recordInfo);
    }

    // and write only if we accept this content
    if (acceptExport) {
        writer.writeRecord(recordInfo);

        totalBytesWritten += byteContent.length;
        entriesCounter++;
    }

}