PHP Functions::save_remote_file_to_local方法代码示例

 private function process_keys_spreadsheet()
 {
     $taxa_objects = array();
     $filename = Functions::save_remote_file_to_local(self::ID_KEYS_FILE, array('timeout' => 4800, 'download_attempts' => 5));
     print "\n[{$filename}]\n";
     foreach (new FileIterator($filename, true) as $line_number => $line) {
         $line = trim($line);
         $fields = explode("\t", $line);
         $name = trim($fields[0]);
         print "\n name: {$name}";
         if ($id_key1 = trim(@$fields[1])) {
             $taxa_objects[$name][] = $id_key1;
         }
         if ($id_key2 = trim(@$fields[2])) {
             $taxa_objects[$name][] = $id_key2;
         }
         if ($id_key3 = trim(@$fields[3])) {
             $taxa_objects[$name][] = $id_key3;
         }
     }
     if (count($taxa_objects) <= 1) {
         echo "\n\nInvalid text file. Program will terminate.\n";
         return;
     }
     return $taxa_objects;
 }

 function get_all_taxa()
 {
     //TODO: [next] button is not processed
     ini_set("auto_detect_line_endings", true);
     $filename = Functions::save_remote_file_to_local($this->strings_to_search, array('cache' => 1, 'resource_id' => '959'));
     //resource_id here is just to have the cache stored in that folder
     $types[1] = 'country records';
     $types[2] = 'US state records';
     $types[3] = 'Canadian province records';
     $i = 1;
     foreach (new FileIterator($filename) as $line_number => $region) {
         if ($region == "") {
             $i++;
             continue;
         }
         // $region = 'China'; //debug
         $type = $types[$i];
         $url = $this->pages[$type] . $region;
         if ($records = self::process_html($url, 'pre')) {
             self::create_archive($records, $region, 'present');
         }
         if ($type == 'country records') {
             $url = $this->pages['endemic'] . $region;
             if ($records = self::process_html($url, 'end')) {
                 self::create_archive($records, $region, 'endemic');
             }
         }
     }
     unlink($filename);
     $this->archive_builder->finalize(TRUE);
 }

 function get_all_taxa($data_dump_url = false)
 {
     $labels = self::get_headers();
     if ($data_dump_url) {
         $this->data_dump_url = $data_dump_url;
     }
     if ($temp_filepath = Functions::save_remote_file_to_local($this->data_dump_url, array('timeout' => 4800, 'download_attempts' => 5))) {
         $not80 = 0;
         $i = 0;
         foreach (new FileIterator($temp_filepath, true) as $line_number => $line) {
             if ($line) {
                 $record = self::prepare_row_data(trim($line), $labels);
                 if (count($record) != 80) {
                     $not80++;
                     // means invalid CSV row, needs attention by provider
                     echo "\n investigate: invalid CSV row, needs attention by provider [" . count($record) . "]";
                     print_r($record);
                 } else {
                     if (@$record['SCIENTIFIC_NAME']) {
                         $i++;
                         debug("{$i}. " . $record['SCIENTIFIC_NAME'] . " [" . count($record) . "]\n");
                         self::parse_record_element($record);
                     }
                 }
             }
         }
         debug("\n not 80: {$not80} \n");
         $this->create_archive();
     }
 }

 private function prepare_files()
 {
     if ($input_file = Functions::save_remote_file_to_local($this->source_file_path, array("cache" => 1, "timeout" => 3600, "file_extension" => "xlsx", 'download_attempts' => 2, 'delay_in_minutes' => 2))) {
         $temp_dir = create_temp_dir() . "/";
         $this->source_file_path = $temp_dir . "spg_falo.txt";
         self::convert_xlsx_to_tab($input_file, $this->source_file_path);
         unlink($input_file);
     }
 }

 /**
  * Download source data from URL to temporary location on local file system.
  */
 private function download_source_data_file()
 {
     $start = microtime(true);
     debug("Downloading source file.");
     $download_options = array('file_extension' => pathinfo($this->source_url, PATHINFO_EXTENSION), 'cache' => true, 'timeout' => 172800);
     $this->source_file_path = Functions::save_remote_file_to_local($this->source_url, $download_options);
     if (!file_exists($this->source_file_path)) {
         throw new \Exception('Error downloading source file.');
     }
     $this->profile($start);
 }

 private function parse_xls()
 {
     if ($this->data_dump_url = Functions::save_remote_file_to_local($this->data_dump_url, array('download_wait_time' => 1000000, 'timeout' => 600, 'download_attempts' => 5, 'file_extension' => 'xls'))) {
         require_library('XLSParser');
         $parser = new XLSParser();
         debug("\n reading: " . $this->data_dump_url . "\n");
         $temp = $parser->convert_sheet_to_array($this->data_dump_url);
         $records = $parser->prepare_data($temp, "single", "SCIENTIFIC NAME", "SCIENTIFIC NAME", "CATEGORY", "ENGLISH NAME", "RANGE", "ORDER", "FAMILY", "EXTINCT", "EXTINCT_YEAR");
         $records = self::fill_in_missing_names($records);
         $records = self::fill_in_parent_id($records);
         debug("\n" . count($records));
         return $records;
     }
 }

 private function parse_xls()
 {
     if ($this->data_dump_url = Functions::save_remote_file_to_local($this->data_dump_url, array('cache' => 1, 'download_wait_time' => 1000000, 'timeout' => 600, 'download_attempts' => 5, 'file_extension' => 'xlsx'))) {
         require_library('XLSParser');
         $parser = new XLSParser();
         debug("\n reading: " . $this->data_dump_url . "\n");
         $temp = $parser->convert_sheet_to_array($this->data_dump_url, 0);
         $records = $parser->prepare_data($temp, "single", "Scientific name", "Scientific name", "Category", "English name", "Range", "Order", "Family", "Extinct", "Extinction Year");
         $records = self::add_uppercase_fields($records);
         $records = self::fill_in_missing_names($records);
         $records = self::fill_in_parent_id($records);
         debug("\n" . count($records));
         return $records;
     }
 }

 function get_all_taxa()
 {
     require_library('XLSParser');
     $docs = count($this->spreadsheets);
     $doc_count = 0;
     foreach ($this->spreadsheets as $doc) {
         $doc_count++;
         echo "\n processing [{$doc}]...\n";
         if ($path = Functions::save_remote_file_to_local($this->url_path . $doc, array("cache" => 1, "timeout" => 3600, "file_extension" => "xls", 'download_attempts' => 2, 'delay_in_minutes' => 2))) {
             $parser = new XLSParser();
             $arr = $parser->convert_sheet_to_array($path);
             $fields = array_keys($arr);
             $i = -1;
             $rows = count($arr["Species"]);
             echo "\n total {$path}: {$rows} \n";
             foreach ($arr["Species"] as $Species) {
                 $i++;
                 $rec = array();
                 foreach ($fields as $field) {
                     $rec[$field] = $arr[$field][$i];
                 }
                 $rec = array_map('trim', $rec);
                 /* breakdown when caching
                    $cont = false;
                    // if($i >= 1 && $i < 6000)         $cont = true;
                    // if($i >= 3000 && $i < 6000)      $cont = true;
                    // if($i >= 6000 && $i < 9000)      $cont = true;
                    // if($i >= 9000 && $i < 12000)     $cont = true;
                    // if($i >= 11800 && $i < 15000)    $cont = true;
                    if(!$cont) continue;
                    */
                 print "\n [{$doc_count} of {$docs}][" . ($i + 1) . " of {$rows}] " . $rec["Species"] . "\n";
                 $rec = self::clean_taxon_name($rec);
                 $taxon_id = trim(preg_replace('/\\s*\\([^)]*\\)/', '', $rec["sciname"]));
                 // remove parenthesis
                 $taxon_id = str_replace(" ", "_", $taxon_id);
                 $rec["taxon_id"] = md5($taxon_id);
                 self::create_instances_from_taxon_object($rec);
                 self::prepare_images($rec);
                 self::prepare_data($rec);
             }
             unlink($path);
         } else {
             echo "\n [{$doc}] unavailable! \n";
         }
     }
     $this->archive_builder->finalize(TRUE);
 }

 function export_xml_to_archive($params, $xml_file_YN = false)
 {
     if (!$xml_file_YN) {
         require_library('connectors/INBioAPI');
         $func = new INBioAPI();
         $paths = $func->extract_archive_file($params["eol_xml_file"], $params["filename"], array("timeout" => 7200, "expire_seconds" => 0));
         // "expire_seconds" -- false => won't expire; 0 => expires now //debug
         print_r($paths);
         $params["path"] = $paths["temp_dir"];
         self::convert_xml($params);
         $this->archive_builder->finalize(TRUE);
         recursive_rmdir($paths["temp_dir"]);
         // remove temp dir
     } else {
         $params['path'] = DOC_ROOT . "tmp/";
         $local_xml_file = Functions::save_remote_file_to_local($params['eol_xml_file'], array('file_extension' => "xml", 'cache' => 0, "timeout" => 7200, "download_attempts" => 2, "delay_in_minutes" => 2));
         //debug - cache should be 0 zero in normal operation
         $params['filename'] = pathinfo($local_xml_file, PATHINFO_BASENAME);
         self::convert_xml($params);
         $this->archive_builder->finalize(TRUE);
         unlink($local_xml_file);
     }
 }

 public function convert_spreadsheet($spreadsheet, $worksheet = null)
 {
     require_library('XLSParser');
     $parser = new XLSParser();
     if ($path = Functions::save_remote_file_to_local($spreadsheet, $this->spreadsheet_options)) {
         $arr = $parser->convert_sheet_to_array($path, $worksheet);
         unlink($path);
         return $arr;
     } else {
         echo "\n [{$spreadsheet}] unavailable! \n";
     }
     return false;
 }

 private function parse_csv_file($type, $taxon = array())
 {
     echo "\n Processing {$type}...\n";
     if ($type == "collection") {
         $no_of_fields = 68;
         if (!in_array(@$taxon["rank"], array("species", "subspecies"))) {
             return;
         }
         $taxon_id = $taxon["orig_no"];
         $url = $this->service[$type] . $taxon["taxon_name"];
         $path = Functions::save_remote_file_to_local($url, $this->download_options);
     } elseif ($type == "occurrence") {
         $no_of_fields = 25;
         if (!in_array(@$taxon["rank"], array("species", "subspecies"))) {
             return;
         }
         $taxon_id = $taxon["orig_no"];
         $url = $this->service[$type] . $taxon["taxon_name"];
         $path = Functions::save_remote_file_to_local($url, $this->download_options);
     } elseif ($type == "taxon") {
         $no_of_fields = 32;
         $path = Functions::save_remote_file_to_local($this->service["taxon"], array("timeout" => 999999, "cache" => 0));
         // debug cache should be 0; only when debugging should be 1
     }
     $j = 0;
     foreach (new FileIterator($path) as $line_number => $line) {
         $rec = array();
         $j++;
         if ($j % 25000 == 0) {
             echo "\n{$j}. [{$type}]";
         }
         // if($j >= 1000) break; //debug
         if ($line) {
             $line = trim($line);
             if ($j == 1) {
                 $fields = explode(",", $line);
                 continue;
             } else {
                 $values = explode(",", $line);
                 $values = str_getcsv($line);
                 if (count($values) == $no_of_fields) {
                     $i = 0;
                     foreach ($values as $value) {
                         $field = str_replace('"', '', $fields[$i]);
                         $rec[$field] = str_replace('"', '', $value);
                         $i++;
                     }
                 } else {
                     print_r($values);
                     echo "\n investigate rec is not {$no_of_fields}";
                 }
             }
             if ($rec) {
                 if ($type == "collection") {
                     self::process_taxon_collection($rec, $taxon_id, $url);
                 } elseif ($type == "occurrence") {
                     self::process_taxon_occurrence($rec, $taxon_id, $url);
                 } elseif ($type == "taxon") {
                     self::process_taxon($rec);
                 }
             }
         }
     }
     unlink($path);
 }

 private function divide_text_file($divisor)
 {
     $temp_filepath = Functions::save_remote_file_to_local(self::DL_MAP_SPECIES_LIST, array('timeout' => 4800, 'download_attempts' => 5));
     if (!$temp_filepath) {
         echo "\n\nExternal file not available. Program will terminate.\n";
         return;
     }
     $i = 0;
     $file_ctr = 0;
     $str = "";
     print "\n";
     foreach (new FileIterator($temp_filepath, true) as $line_number => $line) {
         if ($line) {
             $line .= "\n";
             // FileIterator removes the carriage-return char
             $i++;
             $str .= $line;
             print "{$i}. {$line}\n";
             if ($i == $divisor) {
                 print "\n";
                 $file_ctr++;
                 $file_ctr_str = Functions::format_number_with_leading_zeros($file_ctr, 3);
                 if (!($OUT = Functions::file_open($this->TEMP_FILE_PATH . "batch_" . $file_ctr_str . ".txt", "w"))) {
                     return;
                 }
                 fwrite($OUT, $str);
                 fclose($OUT);
                 $str = "";
                 $i = 0;
             }
         }
         //if($i >= 5) break; //debug
     }
     //last writes
     if ($str) {
         $file_ctr++;
         $file_ctr_str = Functions::format_number_with_leading_zeros($file_ctr, 3);
         if (!($OUT = Functions::file_open($this->TEMP_FILE_PATH . "batch_" . $file_ctr_str . ".txt", "w"))) {
             return;
         }
         fwrite($OUT, $str);
         fclose($OUT);
     }
     //create work_list
     $str = "";
     for ($i = 1; $i <= $file_ctr; $i++) {
         $str .= "batch_" . Functions::format_number_with_leading_zeros($i, 3) . "\n";
     }
     $filename = $this->TEMP_FILE_PATH . "work_list.txt";
     if ($fp = Functions::file_open($filename, "w")) {
         fwrite($fp, $str);
         fclose($fp);
     }
 }

 private function get_uris()
 {
     $uris = array();
     $options = $this->download_options;
     $options["cache"] = 1;
     // $options["expire_seconds"] = 0;
     if ($filename = Functions::save_remote_file_to_local($this->uri_list, $options)) {
         foreach (new FileIterator($filename) as $line_number => $line) {
             if ($line) {
                 $arr = explode("--", $line);
                 if (count($arr) > 1) {
                     $measurement = strtolower(trim(str_ireplace(array(":", "-"), "", $arr[0])));
                     $value = strtolower(trim(str_ireplace(array(":", "-"), "", $arr[1])));
                     $uris[$measurement] = $value;
                 }
             }
         }
         unlink($filename);
     }
     return $uris;
 }

 private function get_vernacular_names()
 {
     $temp_filepath = Functions::save_remote_file_to_local($this->vernacular_path, array('timeout' => 4800, 'download_attempts' => 5));
     foreach (new FileIterator($temp_filepath, true) as $line_number => $line) {
         if ($line) {
             $fields = explode("\t", trim($line));
             $fields = array_map('trim', $fields);
             //trims all array values in the array
             $common_name = @$fields[1];
             $sciname = Functions::canonical_form(trim(@$fields[0]));
             $taxon_id = @$this->taxa_all[$sciname]['Identifier'];
             if ($common_name == '' || $taxon_id == '' || $sciname == '') {
                 continue;
             }
             $language = self::get_language(@$fields[3]);
             $vernacular = new \eol_schema\VernacularName();
             $vernacular->taxonID = $taxon_id;
             $vernacular->vernacularName = (string) $common_name;
             $vernacular->language = $language;
             $vernacular_id = md5("{$vernacular->taxonID}|{$vernacular->vernacularName}|{$vernacular->language}");
             if (!isset($this->vernacular_name_ids[$vernacular_id])) {
                 $this->archive_builder->write_object_to_file($vernacular);
                 $this->vernacular_name_ids[$vernacular_id] = 1;
             }
         }
     }
 }

 private function get_rows_from_dump_file($url)
 {
     $path = Functions::save_remote_file_to_local($url, $this->download_options);
     $urls = array();
     foreach (new FileIterator($path) as $line_number => $line) {
         if ($line) {
             $urls[$line] = "";
         }
     }
     unlink($path);
     return $urls;
 }