Java CrawljaxConfigurationBuilder.addPlugin方法代码示例

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
public static void main(String[] args) {

		CrawljaxConfigurationBuilder builder =
		        CrawljaxConfiguration.builderFor("http://demo.crawljax.com/");
		builder.addPlugin(new OnNewStatePlugin() {

			@Override
			public void onNewState(CrawlerContext context, StateVertex newState) {
				// This will print the DOM when a new state is detected. You should see it in your
				// console.
				LOG.info("Found a new dom! Here it is:\n{}", context.getBrowser().getStrippedDom());
			}

			@Override
			public String toString() {
				return "Our example plugin";
			}
		});
		CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
		crawljax.call();
	}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Run this method to start the crawl.
 */
public static void main(String[] args) {
	CrawljaxConfigurationBuilder builder =
	        CrawljaxConfiguration.builderFor("http://demo.crawljax.com/");

	// Add the invariant that checks that the string isn't present.
	builder.crawlRules().addInvariant(
	        "Detect a string",
	        new NotRegexCondition(
	                "Invariants can be used to perform tests on the current state"));

	// This plugin will just print the error.
	builder.addPlugin(new OnInvariantViolationPlugin() {

		@Override
		public void onInvariantViolation(Invariant invariant, CrawlerContext context) {
			LOG.error("\n\n!!! Invariant {} violated !!!\n", invariant);
		}

	});

	CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
	crawljax.call();
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Entry point
 */
public static void main(String[] args) {
	CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(URL);
	builder.crawlRules().insertRandomDataInInputForms(false);

	builder.crawlRules().click("a");
	builder.crawlRules().click("button");

	// except these
	builder.crawlRules().dontClick("a").underXPath("//DIV[@id='guser']");
	builder.crawlRules().dontClick("a").withText("Language Tools");

	// limit the crawling scope
	builder.setMaximumStates(MAX_NUMBER_STATES);
	builder.setMaximumDepth(MAX_DEPTH);

	builder.addPlugin(new SamplePlugin());

	builder.crawlRules().setInputSpec(getInputSpecification());

	CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
	crawljax.call();
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Add the plugins to the given crawljaxConfiguration.
 * 
 * @param crawljaxConfiguration
 *            the configuration to add the plugins to.
 */
protected static void addPlugins(CrawljaxConfigurationBuilder crawljaxConfiguration) {
	crawljaxConfiguration.addPlugin(new PostCrawlStateGraphChecker());

	crawljaxConfiguration.addPlugin(new OnInvariantViolationPlugin() {

		@Override
		public void onInvariantViolation(Invariant invariant, CrawlerContext context) {
			LargeTestBase.violatedInvariants.add(invariant);
			if (context.getBrowser().getStrippedDom().contains(INVARIANT_TEXT)) {
				violatedInvariantStateIsCorrect = true;
				LOG.warn("Invariant violated: " + invariant.getDescription());
			}
		}

	});
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Override this method to specify a different configuration.
 * 
 * @return a new {@link CrawljaxConfiguration} to crawl with.
 */
protected CrawljaxConfigurationBuilder newCrawlConfigurationBuilder() {
	CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(getUrl());
	builder.crawlRules().clickDefaultElements();
	builder.setUnlimitedRuntime();
	builder.setUnlimitedCrawlDepth();
	builder.addPlugin(new PostCrawlStateGraphChecker());
	return builder;
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
@Before
public void setup() {

	CrawljaxConfigurationBuilder builder = SERVER.newConfigBuilder("crawler/index.html");
	builder.addPlugin(new PreStateCrawlingPlugin() {

		@Override
		public void preStateCrawling(CrawlerContext session,
		        ImmutableList<CandidateElement> candidateElements, StateVertex state) {
			for (CandidateElement candidate : candidateElements) {
				HTMLAnchorElementImpl impl = (HTMLAnchorElementImpl) candidate.getElement();
				impl.setName("fail");
				impl.setId("eventually");
				impl.setHref("will");
				impl.setTextContent("This");
				candidate.getIdentification().setValue("/HTML[1]/BODY[1]/FAILED[1]/A[1]");
			}
		}
	});
	builder.addPlugin(new OnFireEventFailedPlugin() {
		@Override
		public void onFireEventFailed(CrawlerContext context, Eventable eventable,
		        List<Eventable> pathToFailure) {
			hits.incrementAndGet();
		}
	});

	controller = new CrawljaxRunner(builder.build());
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Set Crawljax configuration here
 * @param builder
 */
private void configureCrawljax(CrawljaxConfigurationBuilder builder) {
	
	CSSCatcher cssCatcher = new CSSCatcher();
	cssCatcher.setOutputFolder(outputFolder + "css/");
	
	builder.addPlugin(new CrawlOverview());
	builder.addPlugin(cssCatcher);
	
	//builder.addPlugin(new LoginPlugin());
	
	//builder.crawlRules().clickDefaultElements();
	//builder.crawlRules().dontClick("input").withAttribute("value", "I don't recognize");
	//builder.crawlRules().click("input").withAttribute("type", "submit");
	//builder.crawlRules().dontClick("a").underXPath("//*[@id='pageFooter']");
	//builder.crawlRules().dontClick("a").underXPath("//*[@id='content']/div/div[2]");
	//System.getProperties().setProperty("webdriver.chrome.driver", "chromedriver.exe");
	//builder.setBrowserConfig(new BrowserConfiguration(BrowserType.CHROME, 2));
	builder.setBrowserConfig(new BrowserConfiguration(BrowserType.FIREFOX, 1));
	builder.crawlRules().insertRandomDataInInputForms(false);
	builder.crawlRules().clickElementsInRandomOrder(false);
	builder.crawlRules().crawlFrames(true);
	builder.crawlRules().dontClick("*");
	
	//com.crawljax.browser.WebDriverBackedEmbeddedBrowser s;
	
	builder.setOutputDirectory(new File(outputFolder + "/crawljax"));
					
	builder.setMaximumDepth(1);
	builder.setMaximumStates(2);
	
	builder.crawlRules().waitAfterReloadUrl(20, TimeUnit.MILLISECONDS);
	builder.crawlRules().waitAfterEvent(200, TimeUnit.MILLISECONDS);
	
}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
/**
 * Run this method to start the crawl.
 * 
 * @throws IOException
 *             when the output folder cannot be created or emptied.
 */
public static void main(String[] args) throws IOException {
	CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(URL);
	builder.crawlRules().insertRandomDataInInputForms(false);
	builder.setMaximumRunTime(100, TimeUnit.SECONDS);


	// click these elements
	builder.crawlRules().clickDefaultElements();
	builder.crawlRules().click("div").withAttribute("class", "clickable");

	// but don't click these
	builder.crawlRules().dontClick("a").withAttribute("class", "ignore");
	builder.crawlRules().dontClick("a").underXPath("//DIV[@id='footer']");

	// Set timeouts
	builder.crawlRules().waitAfterReloadUrl(WAIT_TIME_AFTER_RELOAD, TimeUnit.MILLISECONDS);
	builder.crawlRules().waitAfterEvent(WAIT_TIME_AFTER_EVENT, TimeUnit.MILLISECONDS);

	// Add a condition that this XPath doesn't exits
	builder.crawlRules().addCrawlCondition("No spans with foo as class",
	        new NotXPathCondition("//*[@class='foo']"));

	// Set some input for fields
	builder.crawlRules().setInputSpec(getInputSpecification());

	// This will generate a nice output in the output directory.
	File outFolder = new File("output");
	if (outFolder.exists()) {
		FileUtils.deleteDirectory(outFolder);
	}
	builder.setOutputDirectory(outFolder);
	builder.addPlugin(new CrawlOverview());

	// We want to use two browsers simultaneously.
	builder.setBrowserConfig(new BrowserConfiguration(BrowserType.FIREFOX, 1));

	CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
	crawljax.call();

}
 

import com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder; //导入方法依赖的package包/类
private CrawljaxConfiguration readConfig(String urlValue, String outputDir) {
	CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(urlValue);

	builder.setOutputDirectory(new File(outputDir));

	BrowserType browser = BrowserType.FIREFOX;
	if (options.specifiesBrowser()) {
		browser = options.getSpecifiedBrowser();
	}

	int browsers = 1;
	if (options.specifiesParallelBrowsers()) {
		browsers = options.getSpecifiedNumberOfBrowsers();
	}
	if (browser == BrowserType.REMOTE) {
		String remoteUrl = options.getSpecifiedRemoteBrowser();
		builder.setBrowserConfig(BrowserConfiguration.remoteConfig(browsers, remoteUrl));
	} else {
		builder.setBrowserConfig(new BrowserConfiguration(browser, browsers));
	}

	if (options.specifiesDepth()) {
		builder.setMaximumDepth(options.getSpecifiedDepth());
	}

	if (options.specifiesMaxStates()) {
		builder.setMaximumStates(options.getMaxStates());
	}

	if (options.requestsCrawlHiddenAnchors()) {
		builder.crawlRules().crawlHiddenAnchors(true);
	}

	configureTimers(builder);

	builder.addPlugin(new CrawlOverview());

	if (options.specifiesClickElements()) {
		builder.crawlRules().click(options.getSpecifiedClickElements());
	} else {
		builder.crawlRules().clickDefaultElements();
	}

	return builder.build();
}