PHP Functions::lookup_with_cache方法代码示例

 private function process_pages($recs)
 {
     foreach ($recs as $rec) {
         // if($rec->title != "42194843") continue; //debug only
         // if($rec->title != "42194845") continue; //debug only
         // if($rec->title != "33870179") continue; //debug only --with copyrightstatus
         // if($rec->title != "13128418") continue; //debug only --with licensor (13128418, 30413122)
         // if($rec->title != "42194845") continue; //debug only --without licensor
         if ($rec->title != "16059324") {
             continue;
         }
         //debug only
         echo "\n" . $rec->title;
         $url = $this->wikipedia_api . "?action=query&titles=" . urlencode($rec->title) . "&format=json&prop=revisions&rvprop=content";
         $json = Functions::lookup_with_cache($url, array('expire_seconds' => true));
         //this expire_seconds should always be true
         $arr = json_decode($json, true);
         foreach (@$arr['query']['pages'] as $page) {
             if ($val = @$page['revisions'][0]['*']) {
                 if ($data = self::parse_wiki_content($val)) {
                     // if(isset($data['Taxa Found in Page (tabular)']['NameConfirmed'])) self::create_archive($data);
                     if (isset($data['Taxa Found in Page']['text'])) {
                         self::create_archive($data);
                     } else {
                         echo "\n[no taxa found for wiki: " . $data['Page Summary']['PageID'] . "]\n";
                     }
                 }
             }
         }
     }
 }

 public static function get_metadata($url)
 {
     $xml = Functions::lookup_with_cache($url, array('validation_regex' => 'xmlns:'));
     $simple_xml = simplexml_load_string($xml);
     $params = array();
     $dcterms = $simple_xml->children("http://dublincore.org/documents/dcmi-terms/");
     $dwc = $simple_xml->children("http://digir.net/schema/conceptual/darwin/2003/1.0");
     $params['source'] = (string) $dcterms->identifier;
     $data_object = $simple_xml->dataObject;
     $dcterms = $data_object->children("http://dublincore.org/documents/dcmi-terms/");
     $params['citation'] = (string) $dcterms->bibliographicCitation;
     $params['identifier'] = (string) $dcterms->identifier;
     $params['data_type'] = "http://purl.org/dc/dcmitype/Text";
     $params['mime_type'] = "text/html";
     $params['license'] = "not applicable";
     $params['agents'] = array();
     foreach ($data_object->agent as $agent) {
         $agent_name = (string) $agent;
         $attr = $agent->attributes();
         $agent_role = (string) @$attr['role'];
         $params['agents'][] = array($agent_name, $agent_role);
     }
     print_r($xml);
     // print_r($params);
     echo "\n\n\n";
 }

 private function get_contributors()
 {
     $allowed = array("Robertson Ross", "Robertson &", "Bryant Kevin", "Cox Carol & Bob", "Garin James");
     if ($html = Functions::lookup_with_cache($this->image_list_page, $this->download_options)) {
         if (preg_match("/<ul id=\"scbar\"(.*?)<\\/ul>/ims", $html, $arr)) {
             if (preg_match_all("/<a href=(.*?)<\\/a>/ims", $arr[1], $arr)) {
                 $lines = array();
                 foreach ($arr[1] as $line) {
                     foreach ($allowed as $str) {
                         if (is_numeric(stripos($line, $str))) {
                             $lines[$line] = '';
                             break;
                         }
                     }
                 }
                 $contributors = array();
                 foreach (array_keys($lines) as $line) {
                     if (preg_match("/\"(.*?)\"/ims", $line, $arr)) {
                         $path = $this->domain . trim($arr[1]);
                     }
                     if (preg_match("/<\\/i>(.*?)xxx/ims", $line . "xxx", $arr)) {
                         $contributors[trim($arr[1])] = $path;
                     }
                 }
                 print_r($contributors);
                 return $contributors;
             }
         }
     }
     return false;
 }

 function get_all_taxa()
 {
     $this->uris = self::get_uris();
     // print_r($this->uris); exit;
     // offset = 927 k while caching...
     $limit = 500;
     $offset = 2552000;
     //orig limit=500 offset=0
     while (true) {
         $url = $this->service['specimen'] . "&limit={$limit}&offset={$offset}";
         if ($contents = Functions::lookup_with_cache($url, $this->download_options)) {
             $json = json_decode($contents);
             // print_r($json); exit();
             $returned = count($json->result->records);
             echo "\ncount: [{$returned}]\n";
             // self::process_specimen_records($json);
             // break;
         }
         $offset += $limit;
         if ($returned < $limit) {
             break;
         }
     }
     // exit;
     $this->archive_builder->finalize(TRUE);
 }

 private function query_kingdom_GGBN_info($kingdom)
 {
     $records = array();
     $rec["source"] = $this->kingdom_service_ggbn . $kingdom;
     $rec["taxon_id"] = $kingdom;
     if ($html = Functions::lookup_with_cache($rec["source"], $this->download_options)) {
         $has_data = false;
         if (preg_match("/<b>(.*?) entries found/ims", $html, $arr) || preg_match("/<b>(.*?) entry found/ims", $html, $arr)) {
             print "\n {$kingdom}: " . $arr[1] . "\n";
             $pages = self::get_number_of_pages($arr[1]);
             print "\n pages to access: [{$pages}]\n";
             for ($i = 1; $i <= $pages; $i++) {
                 echo "\n {$i} of {$pages} ";
                 if ($i > 1) {
                     $rec["source"] = $this->kingdom_service_ggbn . $kingdom . "&page={$i}";
                     $html = Functions::lookup_with_cache($rec["source"], $this->download_options);
                 }
                 if ($temp = self::process_html($html, $rec["source"])) {
                     $records = array_merge($records, $temp);
                 }
             }
         }
     }
     self::create_instances_from_taxon_object($records);
 }

 private function process_html($url, $type)
 {
     if ($html = Functions::lookup_with_cache($url, $this->download_options)) {
         $html = self::clean_html($html);
         $html = str_ireplace('<td align=center>', '<td>', $html);
         return self::parse_page($html, $type);
     }
     return false;
 }

 function convert_to_dwca($params)
 {
     require_library('FlickrAPI');
     $auth_token = NULL;
     // if(FlickrAPI::valid_auth_token(FLICKR_AUTH_TOKEN)) $auth_token = FLICKR_AUTH_TOKEN;
     $page = 1;
     $per_page = 500;
     $url = $this->service['photosets'] . '&photoset_id=' . $params['photoset_id'] . '&user_id=' . $params['flickr_user_id'] . '&per_page=' . $per_page;
     if ($json = Functions::lookup_with_cache($url . '&page=' . $page, $this->download_options)) {
         $json = str_replace("\\'", "'", $json);
         $obj = json_decode($json);
         $total_pages = ceil($obj->photoset->total / $per_page);
         echo "\ntotal_pages = {$total_pages}\n";
         for ($i = 1; $i <= $total_pages; $i++) {
             if ($json = Functions::lookup_with_cache($url . '&page=' . $page, $this->download_options)) {
                 $json = str_replace("\\'", "'", $json);
                 $obj = json_decode($json);
                 $k = 0;
                 $total_photos = count($obj->photoset->photo);
                 foreach ($obj->photoset->photo as $rec) {
                     $k++;
                     echo "\n{$i} of {$total_pages} - {$k} of {$total_photos}";
                     if (!($sciname = self::get_sciname_from_title($rec->title))) {
                         continue;
                     }
                     // if($sciname == "SONY DSC") //debug
                     // {
                     //     print_r($rec);
                     // }
                     $photo_response = FlickrAPI::photos_get_info($rec->id, $rec->secret, $auth_token, $this->download_options);
                     $photo = @$photo_response->photo;
                     if (!$photo) {
                         continue;
                     }
                     if ($photo->visibility->ispublic != 1) {
                         continue;
                     }
                     if ($photo->usage->candownload != 1) {
                         continue;
                     }
                     if (@(!$GLOBALS["flickr_licenses"][$photo->license])) {
                         continue;
                     }
                     $data_objects = FlickrAPI::get_data_objects($photo, $params['flickr_user_id']);
                     foreach ($data_objects as $do) {
                         self::create_archive($sciname, $do);
                     }
                 }
             }
             $page++;
             // break; //debug
         }
     }
     $this->archive_builder->finalize(TRUE);
 }

 private function parse_html($url)
 {
     $final = array();
     if ($html = Functions::lookup_with_cache($url, $this->download_options)) {
         $html = str_ireplace("APPENDIX: FIRE REGEIME TABLE", "APPENDIX: FIRE REGIME TABLE", $html);
         $final['source'] = $url;
         $final['life_form'] = self::get_Raunkiaer_life_form($html, $url);
         $final['authorship_citation'] = self::get_authorship_citation($html);
         if (preg_match("/<a name=\"AppendixFireRegimeTable\"(.*?)<a name=\"AppendixB\">/ims", $html, $arr) || preg_match("/<a name='AppendixFireRegimeTable'(.*?)<a name='AppendixB'>/ims", $html, $arr) || preg_match("/<a name='APPENDIX: FIRE REGIME TABLE'(.*?)<a name='REFERENCES'>/ims", $html, $arr) || preg_match("/<a name=\"APPENDIX: FIRE REGIME TABLE\"(.*?)<a name=\"REFERENCES\">/ims", $html, $arr) || preg_match("/<a name=\"APPENDIX: FIRE REGIME TABLE\"(.*?)<a name='REFERENCES'>/ims", $html, $arr) || preg_match("/<a name=\"AppendixFireRegimeTable\"(.*?)<a name='REFERENCES'>/ims", $html, $arr) || preg_match("/<a name='AppendixFireRegimeTable'(.*?)<a name='REFERENCES'>/ims", $html, $arr)) {
             if (preg_match_all("/<tr>(.*?)<\\/tr>/ims", $arr[1], $arr2)) {
                 $TRs = $arr2[1];
                 $i = 0;
                 foreach ($TRs as $tr) {
                     $i++;
                     if ($i == 1) {
                         continue;
                     }
                     //exclude first <tr>
                     if (preg_match_all("/<td(.*?)<\\/td>/ims", $tr, $arr3)) {
                         $temp = $arr3[1];
                         $exclude = array(">Vegetation Community", ">Percent of fires", ">Surface or low", ">Mixed<", "vegetation communities");
                         if (self::needle_occurs_in_this_haystack($temp[0] . "<", $exclude)) {
                             continue;
                         }
                         if (count($temp) == 1) {
                             $index = self::clean_html(strip_tags("<td" . $temp[0]));
                         } else {
                             if (isset($index)) {
                                 if ($to_be_added = self::get_term_to_be_added($temp[0])) {
                                     /* // a good way to catch/debug 
                                        if($to_be_added == "Pacific Northwest")
                                        {
                                            print_r($temp);
                                            echo "\nindex[$index]\n";
                                        }
                                        */
                                     if (isset($final['habitat'][$index])) {
                                         if (!in_array($to_be_added, @$final['habitat'][$index])) {
                                             @($final['habitat'][$index][] = $to_be_added);
                                         }
                                     } else {
                                         @($final['habitat'][$index][] = $to_be_added);
                                     }
                                 }
                             }
                         }
                     }
                 }
             }
             // else echo "\n No <tr>s\n";
         }
         // else echo "\nAPPENDIX: FIRE REGIME TABLE not found\n";
     }
     return $final;
 }

 function start()
 {
     $topics = array("About the EoE", "Agricultural & Resource Economics", "Biodiversity", "Biology", "Climate Change", "Ecology", "Environmental & Earth Science", "Energy", "Environmental Law & Policy", "Environmental Humanities", "Food", "Forests", "Geography", "Hazards & Disasters", "Health", "Mining & Materials", "People", "Physics & Chemistry", "Pollution", "Society & Environment", "Water", "Weather & Climate", "Wildlife");
     // $topics = array("Biodiversity");
     foreach ($topics as $topic) {
         $this->count = array();
         //it initializes every topic
         if ($OUT = Functions::file_open($this->html_dir . str_replace(" ", "_", $topic) . ".html", "w")) {
         } else {
             exit("\nFile access problem.\n");
         }
         $url = $this->search_url . "&q={$topic}";
         if ($html = Functions::lookup_with_cache($url, $this->download_options)) {
             if (preg_match("/page 1 of (.*?)<\\/title>/ims", $html, $arr)) {
                 $count = $arr[1];
                 for ($i = 1; $i <= $count; $i++) {
                     if ($html = Functions::lookup_with_cache($url . "&page={$i}", $this->download_options)) {
                         if (preg_match_all("/<h1>(.*?)<\\/h1>/ims", $html, $arr)) {
                             print_r($arr[1]);
                             foreach ($arr[1] as $t) {
                                 if (preg_match("/>(.*?)<\\/a>/ims", $t, $arr2)) {
                                     $new_link_text = $arr2[1];
                                     $word_count = str_word_count($new_link_text);
                                     if ($word_count < 3) {
                                         $new_link_text .= " ({$topic})";
                                     }
                                     //--------------
                                     @$this->count[$new_link_text]++;
                                     /* previous 
                                        if($word_count < 3) $c = ($this->count[$new_link_text] > 1 ? $this->count[$new_link_text] : ''); //ternary
                                        else                $c = "";
                                        */
                                     $c = $this->count[$new_link_text] > 1 ? $this->count[$new_link_text] : '';
                                     //ternary
                                     //--------------
                                     $t = str_replace($arr2[1], $new_link_text . " {$c}", $t);
                                 }
                                 fwrite($OUT, $t . "<br>");
                             }
                             // exit;
                         }
                     }
                 }
             }
         }
         fclose($OUT);
     }
 }

 function __construct($type, $params)
 {
     // exit("\n[$value]\n");
     if ($type == 'usercontrib') {
         $namespace['ForReview'] = 5000;
         $namespace['Published'] = 0;
         $url = $params['server'] . "/StudentContributions/api.php?action=query&list=usercontribs&ucuser=" . $params['user'] . "&uclimit=100&ucdir=older&format=json&ucnamespace=" . $namespace[$params['article_type']] . "&ucshow=top";
         $json = Functions::lookup_with_cache($url, array('expire_seconds' => 0));
         $arr = json_decode($json);
         $titles = array();
         foreach ($arr->query->usercontribs as $item) {
             $titles[] = array('page_title' => $item->title, 'server' => $params['server']);
         }
         $this->body = implode(array_map('api_reader_controller::render_page_row', $titles));
     }
 }

 function extract_archive_file($dwca_file, $check_file_or_folder_name, $download_options = array('timeout' => 172800, 'expire_seconds' => 0))
 {
     debug("Please wait, downloading resource document...");
     $path_parts = pathinfo($dwca_file);
     $filename = $path_parts['basename'];
     $temp_dir = create_temp_dir() . "/";
     debug($temp_dir);
     if ($file_contents = Functions::lookup_with_cache($dwca_file, $download_options)) {
         $temp_file_path = $temp_dir . "" . $filename;
         debug("temp_dir: {$temp_dir}");
         debug("Extracting... {$temp_file_path}");
         if (!($TMP = Functions::file_open($temp_file_path, "w"))) {
             return;
         }
         fwrite($TMP, $file_contents);
         fclose($TMP);
         sleep(5);
         if (preg_match("/^(.*)\\.(tar.gz|tgz)\$/", $dwca_file, $arr)) {
             $cur_dir = getcwd();
             chdir($temp_dir);
             shell_exec("tar -zxvf {$temp_file_path}");
             chdir($cur_dir);
             $archive_path = str_ireplace(".tar.gz", "", $temp_file_path);
         } elseif (preg_match("/^(.*)\\.(zip)\$/", $dwca_file, $arr) || preg_match("/mcz_for_eol(.*?)/ims", $dwca_file, $arr)) {
             shell_exec("unzip -ad {$temp_dir} {$temp_file_path}");
             $archive_path = str_ireplace(".zip", "", $temp_file_path);
         } else {
             debug("-- archive not gzip or zip. [{$dwca_file}]");
             return;
         }
         debug("archive path: [" . $archive_path . "]");
     } else {
         debug("Connector terminated. Remote files are not ready.");
         return;
     }
     if (file_exists($temp_dir . $check_file_or_folder_name)) {
         return array('archive_path' => $temp_dir, 'temp_dir' => $temp_dir);
     } elseif (file_exists($archive_path . "/" . $check_file_or_folder_name)) {
         return array('archive_path' => $archive_path, 'temp_dir' => $temp_dir);
     } else {
         debug("Can't extract archive file. Program will terminate.");
         return;
     }
 }

 function check_if_image_is_broken()
 {
     $options = array('download_wait_time' => 1000000, 'timeout' => 900, 'download_attempts' => 1);
     // 15mins timeout
     $broken = array();
     for ($i = 1; $i <= 58; $i++) {
         $url = "http://eol.org/collections/94950/images?page={$i}&sort_by=3&view_as=3";
         $html = Functions::lookup_with_cache($url, $options);
         echo "\n{$i}. [{$url}]";
         // <a href="/data_objects/26326917"><img alt="84925_88_88" height="68" src="http://media.eol.org/content/2013/09/13/13/84925_88_88.jpg" width="68" /></a>
         if (preg_match_all("/<a href=\"\\/data_objects\\/(.*?)<\\/a>/ims", $html, $arr)) {
             $rows = $arr[1];
             $total_rows = count($rows);
             $k = 0;
             foreach ($rows as $row) {
                 $k++;
                 echo "\n{$i} of 58 - {$k} of {$total_rows}";
                 if (preg_match("/_xxx(.*?)\"/ims", "_xxx" . $row, $arr)) {
                     $id = $arr[1];
                 }
                 if (preg_match("/src=\"(.*?)\"/ims", "_xxx" . $row, $arr)) {
                     $url = $arr[1];
                     $options['cache_path'] = "/Volumes/Eli blue/eol_cache_2/";
                     if ($html = Functions::lookup_with_cache($url, $options)) {
                         echo "\nexists:[{$url}]";
                     } else {
                         echo "\nbroken: [{$url}]";
                         $broken[$id] = $url;
                     }
                     unset($options['cache_path']);
                 }
             }
         }
         // if($i >= 3) break; //debug
     }
     print_r($broken);
 }

 function get_list_of_user_ids()
 {
     // return array("30860816", "5810611"); // Laura F. [5810611], Eli Agbayani [30860816] , (User: [70505] - Ben Fawkes) has 100+ audio files
     $user_ids = array();
     debug("\n Getting all members... " . $this->EOL_members);
     $offset = 0;
     while (true) {
         if ($json = Functions::lookup_with_cache($this->EOL_members . "&offset={$offset}", $this->download_options)) {
             $offset += 50;
             $users = json_decode($json);
             debug("\n members: " . count($users));
             if (!$users) {
                 break;
             }
             foreach ($users as $user) {
                 $user_ids[(string) $user->id] = 1;
             }
         } else {
             debug("\n Connector terminated. Down: " . $this->EOL_members . "\n");
             return array();
         }
     }
     return array_keys($user_ids);
 }

 private function get_usage_key_again($sciname)
 {
     if ($json = Functions::lookup_with_cache($this->gbif_taxon_info . $sciname . "&verbose=true", $this->download_options)) {
         $usagekeys = array();
         $options = array();
         $json = json_decode($json);
         if (!isset($json->alternatives)) {
             return false;
         }
         foreach ($json->alternatives as $rec) {
             if ($rec->canonicalName == $sciname) {
                 $options[$rec->rank][] = $rec->usageKey;
                 $usagekeys[] = $rec->usageKey;
             }
         }
         if ($options) {
             /* from NCBIGGIqueryAPI.php connector
                if(isset($options["FAMILY"])) return min($options["FAMILY"]);
                else return min($usagekeys);
                */
             return min($usagekeys);
         }
     }
     return false;
 }

 private function get_activities()
 {
     $items = array();
     if ($html = Functions::lookup_with_cache($this->path['activities'], $this->download_options)) {
         //manual adjustment
         $html = self::clean_str(functions::remove_whitespace($html));
         $html = str_ireplace('insect visitors</FONT></P>', 'insect visitors<BR><BR>', $html);
         $html = str_ireplace('<P ALIGN="LEFT"><FONT FACE="Times New Roman">prf', '<BR><BR>prf', $html);
         $html = strip_tags($html, "<BR>");
         if (preg_match_all("/<BR>(.*?)<BR>/ims", $html, $arr)) {
             foreach ($arr[1] as $item) {
                 if (preg_match("/xxx(.*?) =/ims", "xxx" . $item, $arr2)) {
                     $items[trim($arr2[1])] = $item;
                 }
             }
         }
     }
     return $items;
 }