PHP scraperwiki::save_var方法代码示例

    $info['against'] = trim(get_first_string($part, 'proti:</span>', '<br')) == '' ? 0 : trim(get_first_string($part, 'proti:</span>', '<br'));
    $info['abstain'] = trim(get_first_string($part, 'zdržel se:</span>', '<br')) == '' ? 0 : trim(get_first_string($part, 'zdržel se:</span>', '<br'));
    $info['number_representatives'] = trim(get_first_string($part, 'Počet zastupitelů:</span>', '<br')) == '' ? 0 : trim(get_first_string($part, 'Počet zastupitelů:</span>', '<br'));
    $info['present'] = trim(get_first_string($part, 'přítomno:</span>', '<br')) == '' ? 0 : trim(get_first_string($part, 'přítomno:</span>', '<br'));
    $trs = $dom->find("table[class=data-grid]", 0)->find("tr");
    array_shift($trs);
    $data = array();
    foreach ($trs as $tr) {
        $tds = $tr->find("td");
        $mp_id = get_first_string($tds[0]->find("a", 0)->href . "&", "memberId=", "&");
        $data[] = array('division_id' => $info['id'], 'mp_id' => $mp_id, 'vote' => trim($tds[1]->plaintext), 'mp_name' => $tds[0]->plaintext);
    }
    //one division done
    scraperwiki::save_sqlite(array('id'), $info, 'division');
    scraperwiki::save_sqlite(array('division_id', 'mp_id'), $data, 'mp_vote');
    scraperwiki::save_var('last_id', $info['id']);
}
/**
 * finds substrings between opening and closing markers
 * @return result array of the substrings
 */
function returnSubstrings($text, $openingMarker, $closingMarker)
{
    $openingMarkerLength = strlen($openingMarker);
    $closingMarkerLength = strlen($closingMarker);
    $result = array();
    $position = 0;
    while (($position = strpos($text, $openingMarker, $position)) !== false) {
        $position += $openingMarkerLength;
        if (($closingMarkerPosition = strpos($text, $closingMarker, $position)) !== false) {
            $result[] = substr($text, $position, $closingMarkerPosition - $position);

 static function save_metadata($metadata_name, $value)
 {
     return scraperwiki::save_var($metadata_name, $value);
 }

foreach ($dom->find("table.contenttable a") as $data) {
    $sectionlinks[] = $data->href;
}
$i = 0;
foreach ($sectionlinks as $link) {
    $sections[] = scraperWiki::scrape("http://www.ruralfinance.org/" . $link);
    $dom = new simple_html_dom();
    $dom->load(end($sections));
    foreach ($dom->find("div[@id='navNext'] a") as $data) {
        $pagetwolinks[] = htmlspecialchars_decode($data->href);
    }
}
$i = 0;
foreach ($pagetwolinks as $link) {
    $it = "";
    $pagetwos[] = scraperWiki::scrape("http://www.ruralfinance.org/" . $link);
    $dom = new simple_html_dom();
    $dom->load(end($pagetwos));
    $j = 0;
    foreach ($dom->find("table.ab tr") as $data) {
        if ($j != 0) {
            $as = $data->find("td a");
            if (strpos($as[0], "http://www.ruralfinance.org/discussion/") > 0) {
                $it .= $as[0] . "<br>";
            }
        }
        $j++;
    }
    scraperwiki::save_var($i, $it);
    $i++;
}

foreach ($entries as $entry) {
    $key = $entry->nodeValue;
}
$num = 1;
preg_match_all('/(\\d{4})/', $key, $matches);
foreach ($matches[0] as $val) {
    scraperwiki::save_var($num, $val);
    //print scraperwiki::get_var('last_page');
    $num++;
}
$xpathExpr2 = '/html/body/form/table/tbody/tr[3]/td[1]/table/tr/td[2]/table[3]';
$entries2 = $xpath->query($xpathExpr2);
foreach ($entries2 as $entry2) {
    $key2 = $entry2->nodeValue;
}
preg_match_all('/(\\d{4})/', $key2, $matches2);
foreach ($matches2[0] as $val) {
    scraperwiki::save_var($num, $val);
    $num++;
}
$xpathExpr3 = '/html/body/form/table/tbody/tr[3]/td[1]/table/tr/td[2]/table[1]';
$entries3 = $xpath->query($xpathExpr3);
foreach ($entries3 as $entry3) {
    $key3 = $entry3->nodeValue;
}
$matches3 = explode(" ", $key3);
scraperwiki::save_var("day", $matches3[2]);
$match = explode("\n", $matches3[1]);
scraperwiki::save_var("date", $match[2]);
//scraperwiki::save_var($num, $val);
//$num++;

    return null;
}
$blacklist = array();
$url = "http://www.coastandcountry.co.uk/cottage-details/";
scraperwiki::attach("coastandcountrycouk");
# get an array of the cottage data to scrape
$cottData = scraperwiki::select("COTTAGE_URL, PRICE_HIGH, PRICE_LOW from 'coastandcountrycouk'.SWDATA order by COTTAGE_URL");
$placeholder = scraperwiki::get_var("cottID");
if ($placeholder != "") {
    $index = searchForId($placeholder, $cottData);
    $cottData = array_splice($cottData, $index);
}
require 'scraperwiki/simple_html_dom.php';
$dom = new simple_html_dom();
foreach ($cottData as $value) {
    scraperwiki::save_var("cottID", $value['COTTAGE_URL']);
    // check the cottage url against the blacklist
    foreach ($blacklist as $blItem) {
        if ($value['COTTAGE_URL'] == $blItem) {
            continue 2;
        }
    }
    //load the page into the scraper
    $html = scraperWiki::scrape($url . $value['COTTAGE_URL']);
    $dom->load($html);
    $feature = "";
    $image = "";
    $imgURL = "";
    $xtraFeatures = "";
    /*  Get the Data  */
    // get Cottage name

    }
    $i++;
}
require_once 'scraperwiki/simple_html_dom.php';
scraperwiki::save_var('dummy', 0);
$maxpages = 46;
$i = 1;
$dom = new simple_html_dom();
$placeholder = scraperwiki::get_var("page");
if ($placeholder) {
    $i = $placeholder;
}
$j = 0;
$counter = 0;
while ($i <= $maxpages) {
    scraperwiki::save_var("page", $i);
    $url = "http://www.norfolkcottages.co.uk/cottage-search/amount-10/page-" . $i;
    // Load HTML from a URL
    $html = file_get_html($url);
    $dom->load($html);
    // get the list of cottages
    #echo $dom;
    // page
    foreach ($dom->find('div[id=search-results-container]') as $page) {
        // cottage
        foreach ($dom->find('div[class=property-result-container]') as $page) {
            foreach ($page->find('div[class=middle-container]') as $cottage) {
                $cottage_name = "";
                # cottage name, URL and ID
                foreach ($cottage->find('h2') as $cottageNameData) {
                    foreach ($cottageNameData->find('a') as $cottageURLData) {

                $mp = trim(substr($td->innertext, 1));
                $club = $h2s[$key]->innertext;
                $votes[] = array('division_id' => $html['id'], 'vote' => $vote, 'mp' => $mp, 'club' => $club);
            }
        }
        //print_r($votes);die();
        scraperwiki::save_sqlite(array('division_id', 'mp'), $votes, 'vote');
        /*$tds = $table->find('td');
            if (count($tds) > 0) {
              foreach ($tds as $td) {
        //echo $td->outertext;
                $h3 = $td->find('h3',0);
                if ($h3 != '') {
                  $party = $h3->innertext;
                } else {
                  $vote = substr($td->innertext,0,1);
                  $mp = trim(substr($td->innertext,1));
                  if ($mp != '') 
                      $votes[] = array(
                        'division_id' => $html['id'],
                        'vote' => $vote,
                        'mp' => $mp,
                        'club' => $party,
                      );
                }
              }
              scraperwiki::save_sqlite(array('division_id','mp'),$votes,'vote');
            } */
        scraperwiki::save_var('last_id', $html['id']);
    }
}

function get_details($url, $CurrentMaxPages)
{
    // get the scraperwiki methods and create a new intance
    require_once 'scraperwiki/simple_html_dom.php';
    $dom = new simple_html_dom();
    // Get the bookmarked page if there is one
    // else start at 1
    $getPage = scraperwiki::get_var("page");
    $page = 1;
    if ($getPage != "") {
        $page = $getPage;
    }
    while ($page <= $CurrentMaxPages) {
        // bookmark record
        scraperwiki::save_var("page", $page);
        //load the page into the scraper
        $html = scraperWiki::scrape($url . $page);
        $dom->load($html);
        // get Details
        $i = 0;
        while ($i < 12) {
            // Get URL
            foreach ($dom->find('a[id=SearchResult1_linkTo_' . $i . ']') as $data) {
                $element = $dom->find('a');
                $cotturl = $data->getAttribute('href');
                $cotturl = str_replace("/cottages/", "", $cotturl);
            }
            // get High / Low Prices
            foreach ($dom->find('span[id=featureBoxPropertyWasPricePoundPr_' . $i . ']') as $data) {
                $prices = str_replace("Prices from ", "", $data->plaintext);
                $prices = str_replace(" based on available 7 nights", "", $prices);
                $prices = str_replace("£", "", $prices);
                $prices = explode("-", $prices);
                $price_low = $prices[0];
                $price_high = $prices[1];
            }
            // Put the records into an array
            $record = array('COTTAGE_URL' => trim($cotturl), 'PRICE_HIGH' => trim($price_high), 'PRICE_LOW' => trim($price_low));
            # save the data
            scraperwiki::save(array('COTTAGE_URL'), $record);
            $i++;
        }
        // move on to the next record
        $page++;
    }
}

    scraperwiki::save_sqlite(array('date'), $data, 'info');
    //print_r($data);
    /*charts*/
    $data_chart = array();
    //html
    $url = 'http://www.ceskatelevize.cz' . $charts_link;
    $html = scraperwiki::scrape($url);
    //get dom
    $dom = new simple_html_dom();
    $dom->load($html);
    $script = $dom->find('script', 2);
    $ar1 = explode(']]', $script->innertext);
    //chart 1
    $ar2 = explode('[[[', str_replace("\t", '', str_replace("\n", '', str_replace(' ', '', $ar1[0]))));
    $ar3 = explode('],[', trim(trim($ar2[1]), ']'));
    foreach ($ar3 as $row) {
        $ar4 = explode(',', $row);
        $data_chart[] = array('date' => $date->format('Y-m-d'), 'chart' => '1', 'minute' => $ar4[0], 'value' => $ar4[1]);
    }
    //chart 2
    $ar2 = explode('[[[', str_replace("\t", '', str_replace("\n", '', str_replace(' ', '', $ar1[1]))));
    $ar3 = explode('],[', trim(trim($ar2[1]), ']'));
    foreach ($ar3 as $row) {
        $ar4 = explode(',', $row);
        $data_chart[] = array('date' => $date->format('Y-m-d'), 'chart' => '2', 'minute' => $ar4[0], 'value' => $ar4[1]);
    }
    scraperwiki::save_sqlite(array('date', 'chart', 'minute'), $data_chart, 'chart');
    //print_r($data_chart);
    scraperwiki::save_var('last_date', $date->format('Y-m-d'));
    $date->add(new DateInterval('P1D'));
}

        $i++;
    }
}
//print_r($data);
scraperwiki::save_var("date", $data[0]);
scraperwiki::save_var("1yearcmt", $data[6]);
$html->__destruct();
# Blank PHP
print "Testing getting 1 year CMT from Bankrate.\n";
$html_content = scraperWiki::scrape("http://www.bankrate.com/rates/interest-rates/1-year-cmt.aspx");
//print $html . "\n";
require 'scraperwiki/simple_html_dom.php';
$html = str_get_html($html_content);
$data[0] = "";
foreach ($html->find("div.interactivetopaction ") as $el) {
    //print $el . "\n";
    //print "1 year CMT ". $el->innertext . "\n";
    $data[0] = $el->innertext;
}
$i = 1;
foreach ($html->find("div.boxcontent") as $box) {
    foreach ($box->find("td") as $el) {
        //print $el->innertext . "\n";
        $data[$i] = $el->innertext;
        $i++;
    }
}
//print_r($data);
scraperwiki::save_var("date", $data[0]);
scraperwiki::save_var("1yearcmt", $data[6]);
$html->__destruct();

function setVariable($key, $value)
{
    scraperwiki::save_var($key, $value);
}

scraperwiki::save_var('cno_max', $MaxCoNo);
//$MaxCoNo = 10;
$MaxDbCoNo = scraperwiki::get_var('cno_dbmax');
scraperwiki::save_var('cno_dbmax_old', $MaxDbCoNo);
//$MaxDbCoNo = 1; // RUN for first time.
$MaxCoNo = 50000;
//it's workaround.
echo $MaxDbCoNo . "->" . $MaxCoNo . "\r\n";
for ($iCom = $MaxDbCoNo; $iCom <= $MaxCoNo; $iCom++) {
    unset($ComData);
    $ComData = GetCommunityData($iCom);
    echo $ComData['cid'] . "/" . $ComData['ctype'] . "\r\n";
    //var_dump($ComData);
    SaveCommunityData($ComData);
    if ($iCom % 50 == 0) {
        scraperwiki::save_var('cno_dbmax', $iCom);
    }
    //break;
}
exit;
function SaveCommunityData($ArrComData)
{
    if (!array_key_exists("cid", $ArrComData)) {
        return;
    }
    scraperwiki::save_sqlite(array("cid"), $ArrComData);
}
function GetMaxCoNumber()
{
    $html = scraperWiki::scrape(URL_NEW);
    $dom = new simple_html_dom();

function scrape_stats()
{
    global $debug;
    $CHI = 0;
    $badCnt = 0;
    $CHI = scraperwiki::get_var('IndxStat', $CHI, "0");
    $churches = scraperwiki::select("church_id from churches");
    //print "Scrape Church Stats @" . $CHI . " of " . sizeof($churches). " " . intval($CHI / sizeof($churches) * 100) . "%\n" ;
    while ($CHI < sizeof($churches)) {
        $church = $churches[$CHI];
        $cid = $church['church_id'];
        if ($debug || $CHI % 100 == 0) {
            print "Stats church Empty=" . $badCnt . " " . $CHI . " of " . sizeof($churches) . " " . intval($CHI / sizeof($churches) * 100) . "%\n";
        }
        if (scrape_one_stat($cid) == 0) {
            $badCnt++;
        }
        //print "Stats@" . $CHI . "/" . $cid . "\n";
        $CHI++;
        scraperwiki::save_var('IndxStat', $CHI);
    }
    print "Scrape Stats Done Churches=" . $CHI . " empty=" . $badCnt . "\n";
    scraperwiki::save_var('IndxStat', 0);
}

        scraperwiki::save_sqlite(array('match_id', 'home_visitor', 'rank'), $goals_visitor, 'goal');
        //cards
        $tr = $table->find('tr', 1);
        $tds = $tr->find('td');
        $yellow_home = str2action($tds[0]->innertext, 'home', $match_id, 'yellow');
        scraperwiki::save_sqlite(array('match_id', 'home_visitor', 'card', 'rank'), $yellow_home, 'card');
        $yellow_visitor = str2action($tds[2]->innertext, 'visitor', $match_id, 'yellow');
        scraperwiki::save_sqlite(array('match_id', 'home_visitor', 'card', 'rank'), $yellow_visitor, 'card');
        $tr = $table->find('tr', 2);
        $tds = $tr->find('td');
        $red_home = str2action($tds[0]->innertext, 'home', $match_id, 'red');
        scraperwiki::save_sqlite(array('match_id', 'home_visitor', 'card', 'rank'), $red_home, 'card');
        $red_visitor = str2action($tds[2]->innertext, 'visitor', $match_id, 'red');
        scraperwiki::save_sqlite(array('match_id', 'home_visitor', 'card', 'rank'), $red_visitor, 'card');
    }
    scraperwiki::save_var('last_id', $match_id);
}
function str2action($str, $home_visitor, $match_id, $card = null)
{
    if ($str != '-') {
        $out = array();
        $rank = 1;
        $goals_ar = explode('<br>', $str);
        foreach ($goals_ar as $goalstr) {
            $fake_dom = new simple_html_dom();
            $fake_dom->load('<html><body>' . $goalstr . '</body></html>');
            if (is_object($fake_dom->find('a', 0))) {
                $min_ar = explode('<', $goalstr);
                $minute = trim(trim($min_ar[0]), '.');
                $player = $fake_dom->find('a', 0)->plaintext;
                $player_link = $fake_dom->find('a', 0)->href;

    // used for debugging
    if ($run_environment == 'dev') {
        if (empty($records)) {
            $records = get_mayors($result);
        } else {
            $records = array_merge($records, get_mayors($result));
        }
    } else {
        get_mayors($result);
        // sleep(10); // this might be needed on scraperwiki.com
    }
    // reset the progress bookmark
    scraperwiki::save_var('last_state', '');
}
// Set state of scraper to complete so we know it didn't crash part way thru
scraperwiki::save_var('scraper_state', 'complete');
// if testing
if ($run_environment == 'dev') {
    header('Content-type: application/json');
    print json_encode($records);
}
function get_post_response($url, $fields)
{
    $fields_string = '';
    //url-ify the data for the POST
    foreach ($fields as $key => $value) {
        $fields_string .= $key . '=' . $value . '&';
    }
    rtrim($fields_string, '&');
    //open connection
    $ch = curl_init();