本文整理汇总了PHP中Crawler类的典型用法代码示例。如果您正苦于以下问题:PHP Crawler类的具体用法?PHP Crawler怎么用?PHP Crawler使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Crawler类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的PHP代码示例。
示例1: go
public function go()
{
$start_url = $this->url;
$c = new Crawler($start_url);
$c->go2linewhere('<p><a href="');
$c->close();
$ledak = explode('<a href="', $c->curline);
for ($i = 1; $i < count($ledak); ++$i) {
$aurl = Crawler::cutuntil($ledak[$i], '"');
$aurl = str_replace('http://hentaifromhell.net/redirect.html?', '', $aurl);
echo "<a href='{$aurl}'>{$aurl}</a><br />\n";
flush();
/*
$basename = Crawler::cutuntillast($aurl, '/');
if (!in_array($basename, $this->blacklist)) {
$c = new Crawler($aurl);
$c->go2linewhere('id="thepic"');
$imgurl = $c->getbetween('SRC="', '"');
$c->close();
echo "<a href='$basename/$imgurl'>".Crawler::n($i,3).".jpg</a><br />\n";
flush();
} else {
echo "$i blacklisted server<br/>";flush();
}
*/
}
}
示例2: getHotSpots
public function getHotSpots()
{
$crawler = new Crawler($this);
$outlines = new CrawlerOutlineCollection();
$size = $this->image->size();
for ($x = 0; $x < $size[0]; $x++) {
for ($y = 0; $y < $size[1]; $y++) {
$pixel = $this->pixel($x, $y);
// Skip white pixels
if ($pixel->color()->compare(ImageColor::white(), 5)) {
continue;
}
// Skip crawled areas
if ($outlines->contains($pixel)) {
continue;
}
// Start crawling
$outline = $crawler->crawl($x, $y);
$outlines->push($outline);
}
}
$hotspots = new ImageCollection();
foreach ($outlines as $outline) {
$hotspots->push($this->image->sliceByOutline($outline));
}
return array($hotspots, $outlines);
}
示例3: crawl_1_page
function crawl_1_page($url)
{
echo "URL2 {$url} <br/>\n";
flush();
$dirname = html_entity_decode(Crawler::cutfromlast1(substr($url, 0, strlen($url) - 1), '/'));
$hasil = array();
$c = new Crawler($url);
$c->go_to('<div class="entry">');
while ($line = $c->readline()) {
if (Crawler::is_there($line, "href='")) {
$img = Crawler::extract($line, "href='", "'");
echo "<a href='{$img}'>{$dirname}</a><br/>\n";
flush();
} else {
if (Crawler::is_there($line, 'href="')) {
$img = Crawler::extract($line, 'href="', '"');
echo "<a href='{$img}'>{$dirname}</a><br/>\n";
flush();
} else {
if (Crawler::is_there($line, '</div>')) {
break;
}
}
}
}
$c->close();
}
示例4: login
/**
* Log the user
* @param Crawler $crawler
*/
private function login($crawler)
{
$form = $crawler->selectButton('_submit')->form();
// définit certaines valeurs
$form['_username'] = 'kwizer';
$form['_password'] = 'sslover';
return $this->client->submit($form);
}
示例5: testCrawl
public function testCrawl()
{
$account = Account::login("searchzen.org", "test");
$c = $account->collections[0];
$crawler = new Crawler($c);
$crawler->pageLimit = 10;
$crawler->start();
}
示例6: crawlForNews
/**
* Start the crawler to retrieve pages from a given news website
* @param type $nrOfDaysBack The nr of days the crawler should go back (counting from today)
* @param type $newsSiteUrl The root URL of the news site (the seed of the crawler)
* @return type
*/
public function crawlForNews($nrOfDaysBack, $newsSiteUrl, $timeToLive, $startDate = null)
{
$crawler = new Crawler($newsSiteUrl, $timeToLive);
if ($startDate) {
$crawler->crawl($nrOfDaysBack, $startDate);
} else {
$crawler->crawl($nrOfDaysBack);
}
return count($crawler->getCrawled());
}
示例7: run
static function run()
{
if (isset($_GET['site_url']) && isset($_GET['sitemap_url']) && CODOF\Access\CSRF::valid($_GET['CSRF_token'])) {
$sitemapObject = new Crawler($_GET['site_url']);
$sitemapPath = ABSPATH . 'sitemap.xml';
$sitemapFile = $sitemapObject->createSitemap($sitemapPath);
// session_write_close();
// ob_end_flush();
exit;
}
}
示例8: crawl_indowebster
function crawl_indowebster($url)
{
//echo "'$url'";
$craw = new Crawler($url);
$craw->go2lineregexor('/(<\\/div><\\/a><\\/div><\\/div>)/', 1, 'href="#idws7"');
$setring = $craw->getbetween('location.href=\'', '\'');
$path = Crawler::extract($setring, 'path=', '&');
$file_orig = Crawler::cutafter($setring, 'file_orig=');
$craw->close();
return '<a href="' . dirname($setring) . '/' . $path . '">' . rawurldecode($file_orig) . '</a>';
}
示例9: mangareader_1_page
public function mangareader_1_page($fil, $url, $prefix, $chapter)
{
$chapter = Crawler::pad($chapter, 3);
$c = new Crawler($fil);
$c->go_to('width="800"');
$img = $c->getbetween('src="', '"');
preg_match('/(\\d+\\.\\w+)$/', basename($img), $m);
$iname = $m[1];
$c->close();
$name = $prefix . '-' . $chapter . '-' . $iname;
return array($name => $img);
}
示例10: addCrawlMap
public function addCrawlMap($src, $patterns)
{
if (!empty($src)) {
$root = $this->_src_root . '/' . $src;
} else {
$root = $this->_src_root;
}
$crawler = new Crawler($root, $patterns);
$paths = $crawler->getPaths();
foreach ($paths as $path) {
$this->addMap($src . '/' . $path, str_replace('site/', '', $path));
}
}
示例11: mangareader_1_page
public function mangareader_1_page($fil, $url, $chapter)
{
$prefix = $this->prefix;
$chapter = Crawler::pad($chapter, 3);
$c = new Crawler($fil);
$c->go_to('width="800"');
$img = $c->getbetween('src="', '"');
// if (@$_GET['show_url']) echo "<a href='$url'>URL</a> ";
preg_match('/(\\d+\\.\\w+)$/', basename($img), $m);
$iname = $m[1];
echo '<li><a href="' . $img . '">' . $prefix . '-' . $chapter . '-' . $iname . '</a>' . "</li>\n";
$c->close();
}
示例12: handleCrawling
public function handleCrawling()
{
$view = new CrawlerResultViewSwe();
if (isset($_SESSION['url'])) {
$url = $_SESSION['url'];
$curl = new Curl();
$crawler = new Crawler($curl, $url);
}
//If user wants to book
if ($view->bookParamExists() && isset($crawler)) {
$group1 = $view->getBookInfo();
$reservationInfo = $crawler->getReservations($group1);
$view->outPutReservationResult($reservationInfo);
//destroy session
$_SESSION = array();
session_destroy();
session_unset();
} else {
if ($view->timeParamExists() && $view->dayParamExists() && $view->movieParamExists() && isset($crawler)) {
$day = $view->getDay();
$time = $view->getTime();
$movie = $view->getMovie();
$dinnerInfo = $crawler->getDinnerInfo($day, $time);
$view->outPutDinnerAlts($dinnerInfo, $time, $movie);
} else {
if ($view->userHasSubmittedURL()) {
//$url ="http://localhost:8080/";
$url = $view->getURL();
$curl = new Curl();
$crawler = new Crawler($curl, $url);
$crawler->getLinks();
$dates = $crawler->getCalendarInfo();
$comparer = new compareData();
$matchingDay = $comparer->compareCommonDates($dates);
if (!is_null($matchingDay)) {
$filmDates = $crawler->getFilmInfo($matchingDay);
$view->outPutMovieResult($filmDates, $matchingDay);
} else {
$view->noMatchingDays();
}
$_SESSION['url'] = $url;
} else {
$view = new formView();
}
}
}
return $view;
}
示例13: testFlickrCrawl
public function testFlickrCrawl() {
$builders = $this->buildData();
$crawler = Crawler::getInstance();
$config = Config::getInstance();
//use fake Flickr API key
$plugin_builder = FixtureBuilder::build('plugins', array('id'=>'2', 'folder_name'=>'flickrthumbnails'));
$option_builder = FixtureBuilder::build('options', array(
'namespace' => OptionDAO::PLUGIN_OPTIONS . '-2',
'option_name' => 'flickr_api_key',
'option_value' => 'dummykey') );
//$config->setValue('flickr_api_key', 'dummykey');
$this->simulateLogin('admin@example.com', true);
$crawler->crawl();
$ldao = DAOFactory::getDAO('LinkDAO');
$link = $ldao->getLinkById(43);
$this->assertEqual($link->expanded_url, 'http://farm3.static.flickr.com/2755/4488149974_04d9558212_m.jpg');
$this->assertEqual($link->error, '');
$link = $ldao->getLinkById(42);
$this->assertEqual($link->expanded_url, '');
$this->assertEqual($link->error, 'No response from Flickr API');
$link = $ldao->getLinkById(41);
$this->assertEqual($link->expanded_url, '');
$this->assertEqual($link->error, 'No response from Flickr API');
}
示例14: tearDown
/**
* Destroy Config, Webapp, $_SESSION, $_POST, $_GET, $_REQUEST
*/
public function tearDown()
{
Config::destroyInstance();
Webapp::destroyInstance();
Crawler::destroyInstance();
if (isset($_SESSION)) {
$this->unsetArray($_SESSION);
}
$this->unsetArray($_POST);
$this->unsetArray($_GET);
$this->unsetArray($_REQUEST);
$this->unsetArray($_SERVER);
$this->unsetArray($_FILES);
Loader::unregister();
$backup_dir = FileDataManager::getBackupPath();
if (file_exists($backup_dir)) {
try {
@exec('cd ' . $backup_dir . '; rm -rf *');
rmdir($backup_dir);
// won't delete if has files
} catch (Exception $e) {
}
}
$data_dir = FileDataManager::getDataPath();
if (file_exists($data_dir . 'compiled_view')) {
try {
@exec('cd ' . $data_dir . '; rm -rf compiled_view');
} catch (Exception $e) {
}
}
parent::tearDown();
}
示例15: control
public function control()
{
$output = "";
$authorized = false;
if (isset($this->argc) && $this->argc > 1) {
// check for CLI credentials
$session = new Session();
$username = $this->argv[1];
if ($this->argc > 2) {
$pw = $this->argv[2];
} else {
$pw = getenv('THINKUP_PASSWORD');
}
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$owner = $owner_dao->getByEmail($username);
if ($owner_dao->isOwnerAuthorized($username, $pw)) {
$authorized = true;
Session::completeLogin($owner);
} else {
$output = "ERROR: Incorrect username and password.";
}
} else {
// check user is logged in on the web
if ($this->isLoggedIn()) {
$authorized = true;
} else {
$output = "ERROR: Invalid or missing username and password.";
}
}
if ($authorized) {
$crawler = Crawler::getInstance();
$crawler->crawl();
}
return $output;
}