PHP Functions::combine_all_eol_resource_xmls方法代码示例

 function start_process($resource_id, $call_multiple_instance)
 {
     $this->resource_id = $resource_id;
     $this->call_multiple_instance = $call_multiple_instance;
     $this->connectors_to_run = 1;
     if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
         if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
             // Divide the big list of ids into small files
             Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
             self::divide_text_file(10000);
             //orig value 10000 debug
             Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
         }
     }
     Functions::process_work_list($this);
     if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
         // Combine all XML files.
         Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_DiscoverLife_batch_*.xml");
         // Set to force harvest
         if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
             $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
         }
         // Delete temp files
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt");
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_DiscoverLife_" . "batch_", "xml");
     }
 }

 function start_process($resource_id, $call_multiple_instance)
 {
     $this->resource_id = $resource_id;
     $this->call_multiple_instance = $call_multiple_instance;
     $this->connectors_to_run = 1;
     if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
         if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
             // Divide the big list of ids into small files
             Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
             $batch = Functions::create_work_list_from_master_file($this->MASTER_LIST, 5000, $this->TEMP_FILE_PATH, "batch_", $this->WORK_LIST);
             //debug orig value 5000
             Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
         }
     }
     Functions::process_work_list($this, $batch);
     if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
         // Combine all XML files.
         Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_Bolds_batch_*.xml");
         // Delete temp files
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt");
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_Bolds_" . "batch_", "xml");
     }
 }

 public function get_all_taxa($resource_id)
 {
     // Delete temp files, possible remnants from interrupted runs
     Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml");
     Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv");
     //divide big file to a more consumable chunks
     $file_count = self::divide_big_csv_file(40000);
     //debug orig is 40000
     if ($file_count === false) {
         return false;
     }
     $all_taxa = array();
     $used_collection_ids = array();
     for ($i = 1; $i <= $file_count; $i++) {
         echo "\nprocessing {$i} => \n";
         $arr = self::get_obis_taxa($this->OBIS_DATA_PATH . "temp_" . $i . ".csv", $used_collection_ids);
         $page_taxa = $arr[0];
         $used_collection_ids = $arr[1];
         $xml = \SchemaDocument::get_taxon_xml($page_taxa);
         $resource_path = $this->OBIS_DATA_PATH . "temp_obis_" . $i . ".xml";
         if (!($OUT = Functions::file_open($resource_path, "w"))) {
             return;
         }
         fwrite($OUT, $xml);
         fclose($OUT);
     }
     // Combine all XML files.
     Functions::combine_all_eol_resource_xmls($resource_id, $this->OBIS_DATA_PATH . "temp_obis_*.xml");
     // Set to force harvest
     if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
         $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
     }
     // Delete temp files
     Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml");
     Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv");
 }

 function start_process($resource_id, $call_multiple_instance, $connectors_to_run = 1)
 {
     $this->resource_id = $resource_id;
     $this->call_multiple_instance = $call_multiple_instance;
     $this->connectors_to_run = $connectors_to_run;
     if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
         if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
             // Divide the big list of ids into small files
             Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
             self::create_master_list();
             Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
         }
     }
     Functions::process_work_list($this);
     if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
         // Combine all XML files.
         Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "sl_batch_*.xml");
         // Set to force harvest
         Functions::set_resource_status_to_force_harvest($resource_id);
         // Delete temp files
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "sl_batch_", "txt");
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "sl_batch_", "xml");
         //debug Don't delete it if you want to check subsets of the resource XML.
     }
 }

 function start_process($resource_id, $call_multiple_instance, $connectors_to_run = 1)
 {
     $this->resource_id = $resource_id;
     $this->call_multiple_instance = $call_multiple_instance;
     $this->connectors_to_run = $connectors_to_run;
     if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
         if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
             Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
             // this will prepare a list of all species id; 13 mins. execution
             self::build_id_list();
             // divides the big list of ids into small files
             self::divide_text_file(10000);
             //debug orig 10000, for testing use 5
             Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
             //remove a task from task list
         }
     }
     Functions::process_work_list($this);
     if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
         // step 3: this should only run when all of instances of step 2 are done
         sleep(10);
         //debug orig 10
         Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_tropicos_batch_*.xml");
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_tropicos_batch_", "xml");
         //debug comment this line if u want to have a source for checking encoding probs in the XML
         Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt");
         Functions::set_resource_status_to_force_harvest($resource_id);
     }
 }

require_library('connectors/ConabioAPI');
$resource_id = 106;
$func = new ConabioAPI();
$func->combine_all_xmls($resource_id);
$resource_path = CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml";
// /* working well - replaces Class='Insecta' to 'Reptilia' if Order=='Squamata' --- WEB-5509
require_library('ResourceDataObjectElementsSetting');
$func = new ResourceDataObjectElementsSetting($resource_id, $resource_path);
$xml = file_get_contents($resource_path);
$xml = $func->replace_taxon_element_value_with_condition("dwc:Class", "Insecta", "Reptilia", $xml, "dwc:Order", "Squamata");
$func->save_resource_document($xml);
// */
// start - this will get Tamborines videos from Vimeo and append it with the main resource 106.xml (DATA-1592)
Functions::file_rename($resource_path, CONTENT_RESOURCE_LOCAL_PATH . "temp_vimeo_to_tamborine1.xml");
get_videos_from_vimeo();
Functions::combine_all_eol_resource_xmls($resource_id, CONTENT_RESOURCE_LOCAL_PATH . "temp_vimeo_to_tamborine*.xml");
unlink(CONTENT_RESOURCE_LOCAL_PATH . "temp_vimeo_to_tamborine1.xml");
unlink(CONTENT_RESOURCE_LOCAL_PATH . "temp_vimeo_to_tamborine2.xml");
// end
if (filesize($resource_path) > 1000) {
    Functions::set_resource_status_to_force_harvest($resource_id);
    Functions::gzip_resource_xml($resource_id);
}
$elapsed_time_sec = time_elapsed() - $timestart;
echo "\n";
echo "elapsed time = " . $elapsed_time_sec / 60 . " minutes \n";
echo "elapsed time = " . $elapsed_time_sec / 60 / 60 . " hours \n";
function get_videos_from_vimeo()
{
    echo "\n -- start access to vimeo ";
    $resource_id = "temp_vimeo_to_tamborine2";

 public function get_all_taxa($resource_id)
 {
     $all_taxa = array();
     $this->used_collection_ids = array();
     $urls = array(FWP_SPECIES_DOC_PATH);
     // you can have multiple sources of the species.xls
     $taxa_arr = self::compile_taxa($urls);
     require_library('XLSParser');
     $parser = new XLSParser();
     $images = self::prepare_table($parser->convert_sheet_to_array(FWP_IMAGES_DOC_PATH), "multiple", "SId", "SId", "PictureId", "dbo_Picture_PictureNote", "PictureType", "IsLegal", "Location", "PicComments", "IsAvailable", "LifeStage", "CollectionName", "CollectionAcronym", "PictureSource", "Surname", "Firstname", "DisplayName", "FileName");
     $comnames = self::prepare_table($parser->convert_sheet_to_array(FWP_COMNAMES_DOC_PATH), "multiple", "SId", "CommonName", "Language");
     $synonyms = self::prepare_table($parser->convert_sheet_to_array(FWP_SYNONYMS_DOC_PATH), "multiple", "SId", "SynGenusSpecies", "SynStatus");
     $i = 1;
     $total = sizeof($taxa_arr);
     $j = 0;
     foreach ($taxa_arr as $taxon_arr) {
         echo "\n {$i} of {$total} -- " . $taxon_arr['SId'];
         $i++;
         $taxon_id = $taxon_arr['SId'];
         $page_taxa = self::get_fishwise_taxa($taxon_arr, @$images[$taxon_id], @$comnames[$taxon_id], @$synonyms[$taxon_id]);
         $all_taxa = array_merge($all_taxa, $page_taxa);
         if ($i % 10000 == 0) {
             $j++;
             $xml = \SchemaDocument::get_taxon_xml($all_taxa);
             $j_str = Functions::format_number_with_leading_zeros($j, 3);
             $resource_path = DOC_ROOT . "/update_resources/connectors/files/FishWisePro/" . $j_str . ".xml";
             if (!($OUT = fopen($resource_path, "w+"))) {
                 debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $resource_path);
                 return;
             }
             fwrite($OUT, $xml);
             fclose($OUT);
             $all_taxa = array();
         }
     }
     if ($all_taxa) {
         $j++;
         $xml = \SchemaDocument::get_taxon_xml($all_taxa);
         $j_str = Functions::format_number_with_leading_zeros($j, 3);
         $resource_path = DOC_ROOT . "/update_resources/connectors/files/FishWisePro/" . $j_str . ".xml";
         if (!($OUT = fopen($resource_path, "w+"))) {
             debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $resource_path);
             return;
         }
         fwrite($OUT, $xml);
         fclose($OUT);
     }
     Functions::combine_all_eol_resource_xmls($resource_id, DOC_ROOT . "/update_resources/connectors/files/FishWisePro/*.xml");
     self::delete_files(DOC_ROOT . "/update_resources/connectors/files/FishWisePro/*.xml");
     return;
 }

 function get_all_taxa($resource_id)
 {
     $data = self::prepare_data();
     if ($data === false) {
         return false;
     }
     $taxa = $data["taxon"];
     $taxon_comnames = $data["taxon_comnames"];
     $taxon_references = $data["taxon_references"];
     $taxon_synonyms = $data["taxon_synonyms"];
     $taxon_dataobject = $data["taxon_dataobject"];
     $GLOBALS['taxon_dataobject_agent'] = $data["taxon_dataobject_agent"];
     $GLOBALS['taxon_dataobject_reference'] = $data["taxon_dataobject_reference"];
     $all_taxa = array();
     $i = 0;
     $total = count(array_keys($taxa));
     $batch = 1000;
     //debug orig 1000
     $batch_count = 0;
     foreach ($taxa as $taxon) {
         // if($taxon["dc_identifier"] != "FB-47873") continue; // debug
         $i++;
         debug("\n{$i} of {$total} " . $taxon["dwc_ScientificName"]);
         $taxon_record["taxon"] = $taxon;
         $taxon_id = $taxon["int_id"];
         $taxon_record["common_names"] = @$taxon_comnames[$taxon_id];
         $taxon_record["references"] = @$taxon_references[$taxon_id];
         $taxon_record["synonyms"] = @$taxon_synonyms[$taxon_id];
         $taxon_record["dataobjects"] = @$taxon_dataobject[$taxon_id];
         $arr = self::get_FishBase_taxa($taxon_record);
         $page_taxa = $arr[0];
         if ($page_taxa) {
             $all_taxa = array_merge($all_taxa, $page_taxa);
         }
         unset($page_taxa);
         if ($i % $batch == 0) {
             $batch_count++;
             $xml = \SchemaDocument::get_taxon_xml($all_taxa);
             $resource_path = $this->TEMP_FILE_PATH . "FB_" . $batch_count . ".xml";
             if (!($OUT = fopen($resource_path, "w"))) {
                 debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $resource_path);
                 return;
             }
             fwrite($OUT, $xml);
             fclose($OUT);
             $all_taxa = array();
         }
     }
     //last batch
     $batch_count++;
     $xml = \SchemaDocument::get_taxon_xml($all_taxa);
     $resource_path = $this->TEMP_FILE_PATH . "FB_" . $batch_count . ".xml";
     if (!($OUT = fopen($resource_path, "w"))) {
         debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $resource_path);
         return;
     }
     fwrite($OUT, $xml);
     fclose($OUT);
     Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "FB_*.xml");
     self::delete_temp_files($this->TEMP_FILE_PATH . "FB_*.xml");
     // remove tmp dir
     $this->TEMP_FILE_PATH = str_ireplace("/fishbase", "", $this->TEMP_FILE_PATH);
     if ($this->TEMP_FILE_PATH) {
         shell_exec("rm -fr {$this->TEMP_FILE_PATH}");
     }
     if ($this->test_run) {
         return $all_taxa;
     }
     //used in testing
 }