Java WebClient类代码示例

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public HiddenHtmlPrompt( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());

        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public static List<String> searchDuck (String keyword) {
    List<String> searchResults = new ArrayList<>();
    try{
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        HtmlPage page = webClient.getPage("https://duckduckgo.com/html/?q=" + keyword);
        List<HtmlAnchor> l = page.getByXPath("//a[@class='result__url']");
        for(HtmlAnchor a: l) {
            searchResults.add(a.getHrefAttribute());
        }


    }
    catch(Exception e){
        System.err.println(e);
    }
    return searchResults;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
private String getXenToken(final Account account) { // TODO: Re-write this if it ever gets used
    final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);

    final HtmlPage page;
    final HtmlInput token;

    account.getCookies().forEach(c -> webClient.getCookieManager().addCookie(c));

    try {
        page = webClient.getPage(account.getForum().getProtocol() + "://" + account.getForum());
        token = page.getFirstByXPath("//*[@id='XenForo']/body/div[1]/aside[2]/div/div/div[1]/div[2]/form/div/input[2]");

        webClient.close();
        return token.getValueAttribute();
    } catch (Exception e) {
        e.printStackTrace();
    }
    webClient.close();
    return null;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public HiddenHtmlAlert( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public HiddenHtmlConfirm( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());

        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
/**
	 * Scraping base experiment. To be used as starting point.
	 *
	 * First things first:
	 * <ol>
	 * <li>
	 * Make sure you're able to retrieve the page. Try again with JS disabled if
	 * the client crashes.
	 * </li>
	 * <li>
	 * Make sure you're actually getting what you want, and that there are no
	 * redirect-, proxy-, or login shenanigans or what not. Have a look at the
	 * retrieved text.
	 * </li>
	 * <li>
	 * Also consider disabling CSS to speed things up a bit.
	 * </li>
	 * </ol>
	 */
	@Test
	public void scrapingBaseExperiment() {
		assertOutputDirectoryExists();

		String url = "http://www.google.com";

		try (WebClient client = new WebClient(BrowserVersion.CHROME)) {
//			client.getOptions().setCssEnabled(false);
//			client.getOptions().setJavaScriptEnabled(false);

			try {
				HtmlPage page = client.getPage(url);
				System.out.println("HtmlPage:");
				System.out.println(page.asText());
				System.out.print("\n");
			} catch (IOException ex) {
				System.err.println("WARNING: failed to visit: " + url);
				ex.printStackTrace(System.err);
			}
		}
	}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public Server()throws IOException{
    browser = new WebClient(BrowserVersion.CHROME);
    problems = new ArrayList<>();
    done = false;
    isSubmitting = false;
    user_name = "XC2";
    pass_word = "donthackme";
    work_list = new ArrayList<>();
    users_data = new HashMap<>();
    application = Executors.newCachedThreadPool();
    //the buffer can be 100 at max (may be adjusted if needed)
    buffer_queue = new LinkedBlockingQueue<>(100);
    
    
    //some browser intitialization to increase efficiency (the cookies part is essential)
    browser.getOptions().setUseInsecureSSL(true);
    browser.getOptions().setJavaScriptEnabled(false);
    browser.getOptions().setCssEnabled(false);
    browser.getOptions().setThrowExceptionOnScriptError(false);
    browser.getCookieManager().setCookiesEnabled(true);
    browser.setAjaxController(new NicelyResynchronizingAjaxController());
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
@Test
public void verifyExampleInClassLevelJavadoc() throws Exception {
	Assume.group(TestGroup.PERFORMANCE);

	WebClient webClient = new WebClient();

	MockMvc mockMvc = MockMvcBuilders.standaloneSetup(TestController.class).build();
	MockMvcWebConnection mockConnection = new MockMvcWebConnection(mockMvc);
	mockConnection.setWebClient(webClient);

	WebRequestMatcher cdnMatcher = new UrlRegexRequestMatcher(".*?//code.jquery.com/.*");
	WebConnection httpConnection = new HttpWebConnection(webClient);
	WebConnection webConnection = new DelegatingWebConnection(mockConnection, new DelegateWebConnection(cdnMatcher, httpConnection));

	webClient.setWebConnection(webConnection);

	Page page = webClient.getPage("http://code.jquery.com/jquery-1.11.0.min.js");
	assertThat(page.getWebResponse().getStatusCode(), equalTo(200));
	assertThat(page.getWebResponse().getContentAsString(), not(isEmptyString()));
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public static void main(String[] args) throws IOException {

        // 浏览器
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setUseInsecureSSL(true);//支持https
        webClient.getOptions().setJavaScriptEnabled(true); // 启用JS解释器，默认为true
        webClient.getOptions().setCssEnabled(false); // 禁用css支持
        webClient.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时，是否抛出异常
        webClient.getOptions().setTimeout(10000); // 设置连接超时时间 ，这里是10S。如果为0，则无限期等待
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.setJavaScriptTimeout(8000);//设置js运行超时时间
        webClient.waitForBackgroundJavaScript(500);//设置页面等待js响应时间,

        // proxy
        //webClient.getOptions().setProxyConfig(new ProxyConfig("IP", 80));

        HtmlPage page = webClient.getPage("http://---");
        String pageXml = page.asXml(); //以xml的形式获取响应文本
        System.out.println(pageXml);

    }
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public static Cookie completeCloudflareBrowserCheck(final String url) {
    WebClient completeClient = new WebClient(BrowserVersion.CHROME);
    completeClient.getOptions().setCssEnabled(false);
    completeClient.getOptions().setThrowExceptionOnFailingStatusCode(false);

    final HtmlPage page;
    final HtmlElement submitButton;
    final HtmlForm challengeForm;

    try {
        page = completeClient.getPage(url);
        completeClient.waitForBackgroundJavaScript(5000);

        submitButton = (HtmlElement) page.createElement("button");
        submitButton.setAttribute("type", "submit");

        challengeForm = (HtmlForm) page.getElementById("challenge-form");
        challengeForm.appendChild(submitButton);
        submitButton.click();

        return completeClient.getCookieManager().getCookie("cf_clearance");
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
public static WebClient create(String host,int port) {
	LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
			"org.apache.commons.logging.impl.NoOpLog");
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(
			Level.OFF);
	java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(
			Level.OFF);

	// LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
	// java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);

	WebClient webClient = new WebClient(BrowserVersion.FIREFOX_17,host,port);
	webClient.getOptions().setUseInsecureSSL(true);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	webClient.getOptions().setCssEnabled(true);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.getOptions().setTimeout(60000);
	webClient.setJavaScriptTimeout(60000);
	webClient.waitForBackgroundJavaScript(120000);
	return webClient;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
private WebDriver getWebClient(int portForJSCoverProxy) {
    Proxy proxy = new Proxy().setHttpProxy("localhost:" + portForJSCoverProxy);
    DesiredCapabilities caps = new DesiredCapabilities();
    caps.setCapability(CapabilityType.PROXY, proxy);
    caps.setJavascriptEnabled(true);
    caps.setBrowserName(BrowserType.HTMLUNIT);
    return new HtmlUnitDriver(caps) {
        @Override
        protected WebClient modifyWebClient(WebClient client) {
            client.setScriptPreProcessor((htmlPage, sourceCode, sourceName, lineNumber, htmlElement) -> {
                if(removeJsSnippets != null && !removeJsSnippets.isEmpty()) {
                    for(String toReplace : removeJsSnippets) {
                        sourceCode = sourceCode.replace(toReplace, "");
                    }
                    return sourceCode;
                } else {
                    return sourceCode;
                }

            });
            return client;
        }
    };
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
private HtmlPage allowAccess(WebClient webClient, HtmlPage allowAccessPage) throws IOException {
	HtmlButton allowAccessButton = (HtmlButton) allowAccessPage.getElementById("submit_approve_access");
	if (allowAccessButton == null) {
		throw new RuntimeException("Cannot find allow access button in html page :\n" + allowAccessPage.asXml());
	}
	webClient.waitForBackgroundJavaScriptStartingBefore(WAIT_DELAY_MS);
	// allowAccessButton.click() does not work because
	// allowAccessButton.isVisible() is false
	// for some reason (click() was working with htmlunit 2.23)
	HtmlPage tokenPage = clickButtonIgnoringVisibility(allowAccessButton);
	return tokenPage;
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
@Before
public void setup() throws IOException {
	webClient = new WebClient();

	DefaultCredentialsProvider creds = new DefaultCredentialsProvider();
	creds.addCredentials("root", "p1");
	webClient.setCredentialsProvider(creds);
	resourceBundle = getBundle("ResourceJSF");
}
 

import com.gargoylesoftware.htmlunit.WebClient; //导入依赖的package包/类
/**
 * HtmlUnit을 이용한 HTML 코드 파싱.
 * @param eachArchiveAddress 실제 만화가 담긴 아카이브 주소
 * @return 성공 시 html 코드를 리턴
 */
private String getHtmlPageHtmlUnit(String eachArchiveAddress) throws Exception {
	/* 필수! 로그 메세지 출력 안함 -> HtmlUnit 이용시 Verbose한 로그들이 너무 많아서 다 끔 */
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF); 
	System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
	
	System.out.print("일반 연결 시도중 ... ");
	
	WebClient webClient = new WebClient();
	webClient.getOptions().setRedirectEnabled(true);
	
	WebRequest req = new WebRequest(new URL(eachArchiveAddress));
	req.setHttpMethod(HttpMethod.POST);
	req.setAdditionalHeader("User-Agent", UserAgent.getUserAgent());
	req.setAdditionalHeader("Accept-Encoding", "gzip"); //20171126 gzip 추가
	req.getRequestParameters().add(new NameValuePair("pass", PASSWORD)); //비밀번호 post 방식 전송
	
	HtmlPage page = webClient.getPage(req);
	
	//Html코드를 포함한 페이지 소스코드가 담길 스트링
	String pageSource = page.asXml();
	
	/** 여기도 페이지 파싱 실패 시 검증하는 코드 들어가야 됨 **/
	
	webClient.close();
	System.out.println("성공");
	return pageSource;
}