本文整理汇总了Java中us.codecraft.webmagic.Request.putExtra方法的典型用法代码示例。如果您正苦于以下问题：Java Request.putExtra方法的具体用法？Java Request.putExtra怎么用？Java Request.putExtra使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类us.codecraft.webmagic.Request的用法示例。
在下文中一共展示了Request.putExtra方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: download
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
@Override
public Page download(Request request, Task task) {
String html = null;
try {
html = casperjs.gatherHtml(new cn.nest.spider.entity.commons.Request(request.getUrl(), true));
} catch(IOException e) {
request.putExtra("EXCEPTION", e);
onError(request);
return null;
}
Page page = new Page().setRawText(html);
page.setRequest(request);
page.setUrl(new PlainText(request.getUrl()));
onSuccess(request);
return page;
}
示例2: download
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
@Override
public Page download(Request request, Task task) {
String html = null;
Site site = null;
if (task != null) {
site = task.getSite();
}
try {
html = casperjs.gatherHtml(new com.gs.spider.model.commons.Request(request.getUrl(), true));
} catch (Exception e) {
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
request.putExtra("EXCEPTION", e);
onError(request);
return null;
}
Page page = new Page();
page.setRawText(html);
page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request);
onSuccess(request);
return page;
}
示例3: test
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
@Ignore("environment depended")
@Test
public void test() {
Task task = new Task() {
@Override
public String getUUID() {
return "1";
}
@Override
public Site getSite() {
return null;
}
@Override
public void cron(String expr) {
}
};
Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/");
request.putExtra("1","2");
redisScheduler.push(request, task);
Request poll = redisScheduler.poll(task);
System.out.println(poll);
}
示例4: test
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
@Ignore("environment depended")
@Test
public void test() {
Task task = new Task() {
@Override
public String getUUID() {
return "1";
}
@Override
public Site getSite() {
return null;
}
};
Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/");
request.putExtra("1","2");
redisScheduler.push(request, task);
Request poll = redisScheduler.poll(task);
assertThat(poll).isEqualTo(request);
}
示例5: writeExceptionLog
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
private void writeExceptionLog(Exception e, Request request) {
if (staticValue.isCommonsSpiderDebug()) {
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
e.printStackTrace(printWriter);
request.putExtra("EXCEPTION", stringWriter.toString());
} else {
request.putExtra("EXCEPTION", e.getMessage());
}
}
示例6: handleResponse
import us.codecraft.webmagic.Request; //导入方法依赖的package包/类
@Override
protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task)
throws IOException {
Page page = null;
try {
page = super.handleResponse(request, charset, httpResponse, task);
} catch(IllegalArgumentException e) {
LOG.warn("响应处理异常， url:{}, 由于 {}", request.getUrl(), e.getLocalizedMessage());
request.putExtra("EXCEPTION", e);
onError(request);
throw e;
}
return page;
}