PHP scraperwiki::get_var方法代码示例

function scrap_yp($last_alphabet = '', $last_page = '')
{
    $alphabet = range('a', 'z');
    if (is_null($last_alphabet) || $last_alphabet == '') {
        $temp_alphabet = scraperwiki::get_var('last_alphabet_loaded');
        if (!is_null($temp_alphabet)) {
            $last_alphabet = $temp_alphabet;
        } else {
            $last_alphabet = 'a';
        }
    }
    if (is_null($last_page) || $last_page == '') {
        $temp_page = scraperwiki::get_var('last_page_loaded');
        if (!is_null($temp_page)) {
            $last_page = $temp_page;
        } else {
            $last_page = 1;
        }
    }
    $yp_base_url = 'http://www.yellowpages.co.id/browse/letter/' . $last_alphabet . '?page=' . $last_page;
    $html = scraperWiki::scrape($yp_base_url);
    $dom = new simple_html_dom();
    $dom->load($html);
    foreach ($dom->find("ul.directory-list") as $data) {
        echo $data;
    }
}

}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Metasonix';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_metasonix'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
    scraperwiki::sqliteexecute("DELETE FROM swdata");
    //Truncate the table before adding new results
}

}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Fairlight';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_fairlight'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
    scraperwiki::sqliteexecute("DELETE FROM swdata");
    //Truncate the table before adding new results
}

 static function get_metadata($metadata_name, $default = null)
 {
     return scraperwiki::get_var($metadata_name, $default);
     //return SW_MetadataClient::create()->get($metadata_name);
 }

}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Jen';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_jen'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
    scraperwiki::sqliteexecute("DELETE FROM swdata");
    //Truncate the table before adding new results
}

<?php

require 'scraperwiki.php';
require 'scraperwiki/simple_html_dom.php';
$startProductId = scraperwiki::get_var("currentId", -1);
if ($startProductId == -1) {
    print "No previous saved position found. Starting from scratch.";
} else {
    print "Resuming from product id {$startProductId}\n";
}
scraperwiki::attach("hobbyking_batteryidlist");
$batteries = scraperwiki::select("id from hobbyking_batteryidlist.data where id > {$startProductId} order by id asc");
$remainingCount = count($batteries);
print "Found {$remainingCount} batteries left to be scraped.";
$maxPerRun = 100;
$loopCount = 0;
foreach ($batteries as $bat) {
    if ($loopCount > $maxPerRun) {
        print "Ending run after {$maxPerRun} iterations.";
        break;
    }
    $productId = $bat['id'];
    print "Retrieving " . $productId . "\n";
    $html = scraperWiki::scrape("http://www.hobbyking.com/hobbyking/store/uh_viewItem.asp?idProduct={$productId}");
    //print $html . "\n";
    $dom = new simple_html_dom();
    $dom->load($html);
    // Get the product data (located in a span tag). Should only be one product data area!
    $productDataAreasDom = $dom->find("SPAN[id=prodDataArea]");
    $productDataDom = $productDataAreasDom[0];
    //print $productData . "\n";

                }
              }
              scraperwiki::save_sqlite(array('division_id','mp'),$votes,'vote');
            } */
        scraperwiki::save_var('last_id', $html['id']);
    }
}
require 'scraperwiki/simple_html_dom.php';
//corrections:
//scraperwiki::save_var('last_id',55626); //55150
/*scraperwiki::sqliteexecute("delete from info where id>55652");
scraperwiki::sqlitecommit();
die();*/
//get last id
//scraperwiki::save_var('last_id',0);
$last_id = scraperwiki::get_var('last_id', 0);
echo $last_id;
//read the saved tables
scraperwiki::attach("cz_senate_voting_records_downloader_2", "src");
$rows = scraperwiki::select("id from src.swdata where id>{$last_id} order by id");
if (!empty($rows)) {
    foreach ($rows as $html) {
        //get dom
        $dom = new simple_html_dom();
        $html2 = scraperwiki::select("* from src.swdata where id={$html['id']}");
        $dom->load(str_replace(" ", " ", $html2[0]['html']));
        //common part
        $div = $dom->find("div[class=wysiwyg]", 0);
        //info
        $h1 = $div->find('h1', 0);
        preg_match('/([0-9]{1,}). schůze/', $h1->innertext, $matches);

}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","Miscellaneous","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Teisco';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_teisco'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
    scraperwiki::sqliteexecute("DELETE FROM swdata");
    //Truncate the table before adding new results
}

        $noidung = $trr->find('div', 0)->innertext;
        //$noidung = utf8_encode($noidung);
        if (mb_strlen($noidung) > 1000) {
            $j++;
            @scraperwiki::save_sqlite(array('id'), array('id' => $j . '-' . $src[0]['url'], 'title' => $src[0]['title'], 'url' => $src[0]['url'], 'content' => base64_encode($noidung), 'order' => $j, 'num' => $src[0]['num'], 'reply' => $src[0]['reply']));
        }
    }
    $html->clear();
    unset($html);
    scraperwiki::save_var('last_id', $i);
}
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach("s-in-s", "src");
//scraperwiki::save_var('last_id', 1);
//exit();
$id = scraperwiki::get_var('last_id');
for ($i = $id; $i < 1900; $i++) {
    $src = scraperwiki::select("* from src.swdata limit {$i},1");
    $url = $src[0]['link'];
    $url = 'http://sexinsex.net/bbs/' . $url;
    $html_content = scraperwiki::scrape($url);
    $html = str_get_html($html_content);
    $data = array();
    $tr = $html->find("div.postmessage div.t_msgfont");
    $j = 0;
    foreach ($tr as $trr) {
        $noidung = $trr->find('div', 0)->innertext;
        //$noidung = utf8_encode($noidung);
        if (mb_strlen($noidung) > 1000) {
            $j++;
            @scraperwiki::save_sqlite(array('id'), array('id' => $j . '-' . $src[0]['url'], 'title' => $src[0]['title'], 'url' => $src[0]['url'], 'content' => base64_encode($noidung), 'order' => $j, 'num' => $src[0]['num'], 'reply' => $src[0]['reply']));

}
function searchForId($id, $array)
{
    foreach ($array as $key => $val) {
        if ($val['COTTAGE_URL'] === $id) {
            return $key;
        }
    }
    return null;
}
$blacklist = array();
$url = "http://www.coastandcountry.co.uk/cottage-details/";
scraperwiki::attach("coastandcountrycouk");
# get an array of the cottage data to scrape
$cottData = scraperwiki::select("COTTAGE_URL, PRICE_HIGH, PRICE_LOW from 'coastandcountrycouk'.SWDATA order by COTTAGE_URL");
$placeholder = scraperwiki::get_var("cottID");
if ($placeholder != "") {
    $index = searchForId($placeholder, $cottData);
    $cottData = array_splice($cottData, $index);
}
require 'scraperwiki/simple_html_dom.php';
$dom = new simple_html_dom();
foreach ($cottData as $value) {
    scraperwiki::save_var("cottID", $value['COTTAGE_URL']);
    // check the cottage url against the blacklist
    foreach ($blacklist as $blItem) {
        if ($value['COTTAGE_URL'] == $blItem) {
            continue 2;
        }
    }
    //load the page into the scraper

                scraperwiki::save(array('id'), $dbdata);
                $html->__destruct();
            }
            //End single room load
        }
        // End the listingnum loop
    }
    // End if
    scraperwiki::save_var('last_page', $i);
    $html->__destruct();
}
// End the  for loop
require 'scraperwiki/simple_html_dom.php';
set_time_limit(0);
// While $i (the loop counter) is less than or equal to 9999 (the number of times you want to navigate)
$lastpage = scraperwiki::get_var('last_page');
//$lastpage = 60;
print $lastpage;
for ($i = $lastpage; $i <= 9999; $i++) {
    $html_content = scraperwiki::scrape("https://www.airbnb.co.uk/s?page=" . $i . "&room_types[]=Entire+home%2Fapt");
    $html = str_get_html($html_content);
    if (!empty($html)) {
        $listingnum = $html->find("li.search_result");
        foreach ($listingnum as $el) {
            //print $el-> . "\n";
            $url_listing = array('listingnum' => $el->getAttribute('data-hosting-id'), 'neighborhood' => isset($el->find('span.neighborhood-link', 0)->plaintext) ? $el->find('span.neighborhood-link', 0)->plaintext : null, 'url' => 'https://www.airbnb.com/rooms/' . $el->getAttribute('data-hosting-id'));
            $neigh = $url_listing['neighborhood'];
            $hostingid = $url_listing['listingnum'];
            $url_listing = $url_listing['url'];
            $url_listing = scraperwiki::scrape($url_listing);
            $html = str_get_html($url_listing);

        border-radius: 15px;
        max-width: 400px;
    }
    .tip ul { margin: 0; padding: 0; text-align:center; }
    .tip ul li{ list-style-type: none; padding: 0; margin: 0; }
    .tip ul li.synth_name { font-size: 20px; }
    .tip ul li.price { font-size: 16px; padding: 5px; color: green; }
    .tip ul li.date { font-size: 14px; color: #e7e7e7; text-align: right; }
    .tip ul li.description { font-size: 14px; color: #e7e7e7; text-align: left; padding-bottom: 15px; }
</style>
<div id="content">
    <h1 align='center'><a href="http://synthfilter.com" title="Synth Filter">SynthFilter</a>: <a href='http://geo.craigslist.org/iso/us/ca' target='_blank'>California</a> - Craigslist</h1>
    <div class='synth-container'>
    <?php 
scraperwiki::attach('synthfilter_utils');
$manufacturers = scraperwiki::get_var('manufacturers');
$manufacturers = explode(',', $manufacturers);
//Convert to array
if (!empty($_GET['start'])) {
    $start = $_GET['start'];
}
// To take care global variable if OFF
if (empty($start)) {
    $start = 0;
}
if (strlen($start) > 0 and !is_numeric($start)) {
    //echo "Data Error";
    //exit;
    $start = 0;
}
$sourcescraper = 'california_craigslist_synth_collector';