Java WebClient.getPage方法代码示例

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public static List<String> searchDuck (String keyword) {
    List<String> searchResults = new ArrayList<>();
    try{
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        HtmlPage page = webClient.getPage("https://duckduckgo.com/html/?q=" + keyword);
        List<HtmlAnchor> l = page.getByXPath("//a[@class='result__url']");
        for(HtmlAnchor a: l) {
            searchResults.add(a.getHrefAttribute());
        }


    }
    catch(Exception e){
        System.err.println(e);
    }
    return searchResults;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
private String getXenToken(final Account account) { // TODO: Re-write this if it ever gets used
    final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);

    final HtmlPage page;
    final HtmlInput token;

    account.getCookies().forEach(c -> webClient.getCookieManager().addCookie(c));

    try {
        page = webClient.getPage(account.getForum().getProtocol() + "://" + account.getForum());
        token = page.getFirstByXPath("//*[@id='XenForo']/body/div[1]/aside[2]/div/div/div[1]/div[2]/form/div/input[2]");

        webClient.close();
        return token.getValueAttribute();
    } catch (Exception e) {
        e.printStackTrace();
    }
    webClient.close();
    return null;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
@Test
public void verifyExampleInClassLevelJavadoc() throws Exception {
	Assume.group(TestGroup.PERFORMANCE);

	WebClient webClient = new WebClient();

	MockMvc mockMvc = MockMvcBuilders.standaloneSetup(TestController.class).build();
	MockMvcWebConnection mockConnection = new MockMvcWebConnection(mockMvc);
	mockConnection.setWebClient(webClient);

	WebRequestMatcher cdnMatcher = new UrlRegexRequestMatcher(".*?//code.jquery.com/.*");
	WebConnection httpConnection = new HttpWebConnection(webClient);
	WebConnection webConnection = new DelegatingWebConnection(mockConnection, new DelegateWebConnection(cdnMatcher, httpConnection));

	webClient.setWebConnection(webConnection);

	Page page = webClient.getPage("http://code.jquery.com/jquery-1.11.0.min.js");
	assertThat(page.getWebResponse().getStatusCode(), equalTo(200));
	assertThat(page.getWebResponse().getContentAsString(), not(isEmptyString()));
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {

        // 浏览器
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setUseInsecureSSL(true);//支持https
        webClient.getOptions().setJavaScriptEnabled(true); // 启用JS解释器，默认为true
        webClient.getOptions().setCssEnabled(false); // 禁用css支持
        webClient.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时，是否抛出异常
        webClient.getOptions().setTimeout(10000); // 设置连接超时时间 ，这里是10S。如果为0，则无限期等待
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.setJavaScriptTimeout(8000);//设置js运行超时时间
        webClient.waitForBackgroundJavaScript(500);//设置页面等待js响应时间,

        // proxy
        //webClient.getOptions().setProxyConfig(new ProxyConfig("IP", 80));

        HtmlPage page = webClient.getPage("http://---");
        String pageXml = page.asXml(); //以xml的形式获取响应文本
        System.out.println(pageXml);

    }
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public static Cookie completeCloudflareBrowserCheck(final String url) {
    WebClient completeClient = new WebClient(BrowserVersion.CHROME);
    completeClient.getOptions().setCssEnabled(false);
    completeClient.getOptions().setThrowExceptionOnFailingStatusCode(false);

    final HtmlPage page;
    final HtmlElement submitButton;
    final HtmlForm challengeForm;

    try {
        page = completeClient.getPage(url);
        completeClient.waitForBackgroundJavaScript(5000);

        submitButton = (HtmlElement) page.createElement("button");
        submitButton.setAttribute("type", "submit");

        challengeForm = (HtmlForm) page.getElementById("challenge-form");
        challengeForm.appendChild(submitButton);
        submitButton.click();

        return completeClient.getCookieManager().getCookie("cf_clearance");
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
/**
 * HtmlUnit을 이용한 HTML 코드 파싱.
 * @param eachArchiveAddress 실제 만화가 담긴 아카이브 주소
 * @return 성공 시 html 코드를 리턴
 */
private String getHtmlPageHtmlUnit(String eachArchiveAddress) throws Exception {
	/* 필수! 로그 메세지 출력 안함 -> HtmlUnit 이용시 Verbose한 로그들이 너무 많아서 다 끔 */
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF); 
	System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
	
	System.out.print("일반 연결 시도중 ... ");
	
	WebClient webClient = new WebClient();
	webClient.getOptions().setRedirectEnabled(true);
	
	WebRequest req = new WebRequest(new URL(eachArchiveAddress));
	req.setHttpMethod(HttpMethod.POST);
	req.setAdditionalHeader("User-Agent", UserAgent.getUserAgent());
	req.setAdditionalHeader("Accept-Encoding", "gzip"); //20171126 gzip 추가
	req.getRequestParameters().add(new NameValuePair("pass", PASSWORD)); //비밀번호 post 방식 전송
	
	HtmlPage page = webClient.getPage(req);
	
	//Html코드를 포함한 페이지 소스코드가 담길 스트링
	String pageSource = page.asXml();
	
	/** 여기도 페이지 파싱 실패 시 검증하는 코드 들어가야 됨 **/
	
	webClient.close();
	System.out.println("성공");
	return pageSource;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
private HtmlPage navigateToPage( User user, String uri, boolean enableJS ) throws Exception, IOException, MalformedURLException
{
    final WebClient webClient = new WebClient();
    webClient.getOptions().setCssEnabled( false );
    webClient.getOptions().setJavaScriptEnabled( enableJS );
    addCredentials( user, webClient );

    String pageUrl = new StringBuilder( "http://" ).append( WEBSITE ).append( uri ).toString();
    HtmlPage page = webClient.getPage( pageUrl );
    return page;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public void cancelBooking( User user, Meeting meeting ) throws Exception
{
    final WebClient webClient = new WebClient();
    addCredentials( user, webClient );

    String uri = CANCEL_BOOKING + "/" + meeting.getMeetingId();
    String pageUrl = new StringBuilder( "http://" ).append( WEBSITE ).append( uri ).toString();

    URL url = new URL( pageUrl );
    WebRequest requestSettings = new WebRequest( url, HttpMethod.POST );

    Page redirectPage = webClient.getPage( requestSettings );
    logger.debug( "Confirmed Cancellation " + redirectPage.getWebResponse().getContentAsString() );
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
    
    // 屏蔽HtmlUnit等系统 log
    LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
    java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
    java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(Level.OFF);
    
    String url = "https://www.douyin.com/share/video/6496703951436516621/?mid=6484356820260686606";
    System.out.println("Loading page now-----------------------------------------------: "+url);
    
    /* HtmlUnit 模拟浏览器 */
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setJavaScriptEnabled(true);              // 启用JS解释器，默认为true  
    webClient.getOptions().setCssEnabled(false);                    // 禁用css支持  
    webClient.getOptions().setThrowExceptionOnScriptError(false);   // js运行错误时，是否抛出异常
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setTimeout(10 * 1000);                   // 设置连接超时时间
    HtmlPage page = webClient.getPage(url);
    webClient.waitForBackgroundJavaScript(30 * 1000);               // 等待js后台执行30秒

    String pageAsXml = page.asXml();
    
    /* Jsoup解析处理 */
    // Document doc = Jsoup.parse(pageAsXml, "https://bluetata.com/");
    Document doc = Jsoup.parse(pageAsXml);  
    Elements pngs = doc.select("img[src$=.png]");                   // 获取所有图片元素集
    // 其他操作
    System.out.println(doc.toString());
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
/**
 * Starts the CSV link creation process.
 * 
 * @throws Exception If there was an error while getting the CSV specific information from the forum thread.
 */
public void create() throws Exception{
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
	
	WebClient webClient = new WebClient(BrowserVersion.CHROME);
    HtmlPage page = (HtmlPage) webClient.getPage(forumThreadLink);
    webClient.waitForBackgroundJavaScriptStartingBefore(waitForJavaScript);
    HtmlAnchor anchor = (HtmlAnchor) page.getByXPath(xPath).get(0);
    csvFileLink = anchor.getHrefAttribute();
    csvFileLink = "http://www.pathofexile.com" + csvFileLink.subSequence(0, csvFileLink.length()-1);
    webClient.close();  
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
@Test
public void getPageWhenUrlIsRelativeAndNoPortWillUseLocalhost8080() throws Exception {
	MockEnvironment environment = new MockEnvironment();
	WebClient client = new LocalHostWebClient(environment);
	WebConnection connection = mockConnection();
	client.setWebConnection(connection);
	client.getPage("/test");
	verify(connection).getResponse(this.requestCaptor.capture());
	assertThat(this.requestCaptor.getValue().getUrl())
			.isEqualTo(new URL("http://localhost:8080/test"));
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
@Test
public void getPageWhenUrlIsRelativeAndHasPortWillUseLocalhostPort()
		throws Exception {
	MockEnvironment environment = new MockEnvironment();
	environment.setProperty("local.server.port", "8181");
	WebClient client = new LocalHostWebClient(environment);
	WebConnection connection = mockConnection();
	client.setWebConnection(connection);
	client.getPage("/test");
	verify(connection).getResponse(this.requestCaptor.capture());
	assertThat(this.requestCaptor.getValue().getUrl())
			.isEqualTo(new URL("http://localhost:8181/test"));
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
/**
 * 通过htmlunit来获得一些搜狗的网址。
 * @param key
 * @return
 * @throws Exception
 */
public String getNextUrl(String key){
	String page = new String();
	try {
		WebClient webClient = new WebClient();
		webClient.getOptions().setCssEnabled(false);
		webClient.getOptions().setJavaScriptEnabled(false);
		
		//去拿网页
		HtmlPage htmlPage = webClient.getPage("http://pic.sogou.com/");
		//得到表单
		HtmlForm form = htmlPage.getFormByName("searchForm");
		//得到提交按钮
		HtmlSubmitInput button = form.getInputByValue("搜狗搜索");
		//得到输入框

		HtmlTextInput textField = form.getInputByName("query");
		//输入内容
		textField.setValueAttribute(key);
		//点一下按钮
		HtmlPage nextPage = button.click();

		String str = nextPage.toString();
		page = cutString(str);
		webClient.close();
	} catch (Exception e) {
		// TODO: handle exception
	} finally{
		
	}
	System.out.println();
	return page;
	
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public void testGetToken() throws FailingHttpStatusCodeException, IOException {
	WebClient wc = weiWS.getInstance();
	WebRequest wr = new WebRequest(new URL("https://mp.weixin.qq.com/cgi-bin/login"), HttpMethod.POST);
	wr.getAdditionalHeaders().put("Accept", "*/*");
	wr.getAdditionalHeaders().put("Accept-Encoding", "gzip, deflate, br");
	wr.getAdditionalHeaders().put("Accept-Language", "en-US,en;q=0.5");
	wr.getAdditionalHeaders().put("Connection", "keep-alive");
	//		wr.getAdditionalHeaders().put("Content-Length", "86");
	wr.getAdditionalHeaders().put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
	wr.getAdditionalHeaders().put("Host", "mp.weixin.qq.com");
	wr.getAdditionalHeaders().put("Referer", "https://mp.weixin.qq.com/");
	wr.getAdditionalHeaders().put("User-Agent",
			"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0");
	wr.getAdditionalHeaders().put("X-Requested-With", "XMLHttpRequest");
	List<NameValuePair> params = new ArrayList<NameValuePair>();
	params.add(new NameValuePair("f", "json"));
	params.add(new NameValuePair("imgcode", ""));
	//3BCEBC34A480F472E27372F9DE2D5592
	params.add(new NameValuePair("pwd", "3bcebc34a480f472e27372f9de2d5592"));
	params.add(new NameValuePair("username", "[email protected]"));
	wr.setRequestParameters(params);
	Page page = wc.getPage(wr);
	String response = page.getWebResponse().getContentAsString();
	System.out.println(response);
	Set<Cookie> cookies = weiWS.getInstance().getCookieManager().getCookies();
	for (Cookie cookie : cookies) {
		System.out.println(cookie.toString());
	}

	Gson gson = new Gson();
	RespWithTokenJSON jsonObj = gson.fromJson(response, RespWithTokenJSON.class);
	String homePath = jsonObj.getRedirect_url();

	HtmlPage homePage = wc.getPage("https://mp.weixin.qq.com/" + homePath);
	HtmlAnchor logout = (HtmlAnchor) homePage.getElementById("logout");
	System.out.println(logout);
	HtmlPage loginPage = wc.getPage("https://mp.weixin.qq.com");
	System.out.println(loginPage.getUrl());
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception{

		WebClient webClient = HtmlUnitUtil.create();
		System.out.println(DateFormatUtils.format(new Date(), "yyyyMMddHHmmss"));
		for(int i=0;i<10;i++){
			webClient.getPage("https://detail.m.tmall.com/item.htm?spm=0.0.0.0.5FpUFm&id=543557417955&abtest=21&rn=bbe5ec2e6fea4193aa04dbb1b8754dfd&sid=eb80b81938bb970e9c61caa81b4baf51");
			System.out.println("当前已刷>>>>>>>>>>>>>>>>>"+i);
		}
		System.out.println(DateFormatUtils.format(new Date(), "yyyyMMddHHmmss"));
	}