PHP FinderIndexerHelper::parse方法代码示例

 /**
  * Method to parse input, tokenize it, and then add it to the database.
  *
  * @param   mixed    $input    String or resource to use as input. A resource
  *                             input will automatically be chunked to conserve
  *                             memory. Strings will be chunked if longer than
  *                             2K in size.
  * @param   integer  $context  The context of the input. See context constants.
  * @param   string   $lang     The language of the input.
  * @param   string   $format   The format of the input.
  *
  * @return  integer  The number of tokens extracted from the input.
  *
  * @since   2.5
  */
 protected function tokenizeToDb($input, $context, $lang, $format)
 {
     $count = 0;
     $buffer = null;
     if (!empty($input)) {
         // If the input is a resource, batch the process out.
         if (is_resource($input)) {
             // Batch the process out to avoid memory limits.
             while (!feof($input)) {
                 // Read into the buffer.
                 $buffer .= fread($input, 2048);
                 /*
                  * If we haven't reached the end of the file, seek to the last
                  * space character and drop whatever is after that to make sure
                  * we didn't truncate a term while reading the input.
                  */
                 if (!feof($input)) {
                     // Find the last space character.
                     $ls = strrpos($buffer, ' ');
                     // Adjust string based on the last space character.
                     if ($ls) {
                         // Truncate the string to the last space character.
                         $string = substr($buffer, 0, $ls);
                         // Adjust the buffer based on the last space for the next iteration and trim.
                         $buffer = JString::trim(substr($buffer, $ls));
                     } else {
                         $string = $buffer;
                     }
                 } else {
                     $string = $buffer;
                 }
                 // Parse the input.
                 $string = FinderIndexerHelper::parse($string, $format);
                 // Check the input.
                 if (empty($string)) {
                     continue;
                 }
                 // Tokenize the input.
                 $tokens = FinderIndexerHelper::tokenize($string, $lang);
                 // Add the tokens to the database.
                 $count += $this->addTokensToDb($tokens, $context);
                 // Check if we're approaching the memory limit of the token table.
                 if ($count > self::$state->options->get('memory_table_limit', 30000)) {
                     $this->toggleTables(false);
                 }
                 unset($string);
                 unset($tokens);
             }
         } elseif (strlen($input) > 2048) {
             $start = 0;
             $end = strlen($input);
             $chunk = 2048;
             /*
              * As it turns out, the complex regular expressions we use for
              * sanitizing input are not very efficient when given large
              * strings. It is much faster to process lots of short strings.
              */
             while ($start < $end) {
                 // Setup the string.
                 $string = substr($input, $start, $chunk);
                 // Find the last space character if we aren't at the end.
                 $ls = $start + $chunk < $end ? strrpos($string, ' ') : false;
                 // Truncate to the last space character.
                 if ($ls !== false) {
                     $string = substr($string, 0, $ls);
                 }
                 // Adjust the start position for the next iteration.
                 $start += $ls !== false ? $ls + 1 - $chunk + $chunk : $chunk;
                 // Parse the input.
                 $string = FinderIndexerHelper::parse($string, $format);
                 // Check the input.
                 if (empty($string)) {
                     continue;
                 }
                 // Tokenize the input.
                 $tokens = FinderIndexerHelper::tokenize($string, $lang);
                 // Add the tokens to the database.
                 $count += $this->addTokensToDb($tokens, $context);
                 // Check if we're approaching the memory limit of the token table.
                 if ($count > self::$state->options->get('memory_table_limit', 30000)) {
                     $this->toggleTables(false);
                 }
             }
         } else {
             // Parse the input.
//.........这里部分代码省略.........

 /**
  * Method to index a content item.
  *
  * @param   FinderIndexerResult  $item    The content item to index.
  * @param   string               $format  The format of the content. [optional]
  *
  * @return  integer  The ID of the record in the links table.
  *
  * @since   3.0
  * @throws  Exception on database error.
  */
 public function index($item, $format = 'html')
 {
     // Mark beforeIndexing in the profiler.
     static::$profiler ? static::$profiler->mark('beforeIndexing') : null;
     $db = JFactory::getDbo();
     $nd = $db->getNullDate();
     // Check if the item is in the database.
     $query = $db->getQuery(true)->select($db->quoteName('link_id') . ', ' . $db->quoteName('md5sum'))->from($db->quoteName('#__finder_links'))->where($db->quoteName('url') . ' = ' . $db->quote($item->url));
     // Load the item  from the database.
     $db->setQuery($query);
     $link = $db->loadObject();
     // Get the indexer state.
     $state = static::getState();
     // Get the signatures of the item.
     $curSig = static::getSignature($item);
     $oldSig = isset($link->md5sum) ? $link->md5sum : null;
     // Get the other item information.
     $linkId = empty($link->link_id) ? null : $link->link_id;
     $isNew = empty($link->link_id) ? true : false;
     // Check the signatures. If they match, the item is up to date.
     if (!$isNew && $curSig == $oldSig) {
         return $linkId;
     }
     /*
      * If the link already exists, flush all the term maps for the item.
      * Maps are stored in 16 tables so we need to iterate through and flush
      * each table one at a time.
      */
     if (!$isNew) {
         for ($i = 0; $i <= 15; $i++) {
             // Flush the maps for the link.
             $query->clear()->delete($db->quoteName('#__finder_links_terms' . dechex($i)))->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
             $db->setQuery($query);
             $db->execute();
         }
         // Remove the taxonomy maps.
         FinderIndexerTaxonomy::removeMaps($linkId);
     }
     // Mark afterUnmapping in the profiler.
     static::$profiler ? static::$profiler->mark('afterUnmapping') : null;
     // Perform cleanup on the item data.
     $item->publish_start_date = (int) $item->publish_start_date != 0 ? $item->publish_start_date : $nd;
     $item->publish_end_date = (int) $item->publish_end_date != 0 ? $item->publish_end_date : $nd;
     $item->start_date = (int) $item->start_date != 0 ? $item->start_date : $nd;
     $item->end_date = (int) $item->end_date != 0 ? $item->end_date : $nd;
     // Prepare the item description.
     $item->description = FinderIndexerHelper::parse($item->summary);
     /*
      * Now, we need to enter the item into the links table. If the item
      * already exists in the database, we need to use an UPDATE query.
      * Otherwise, we need to use an INSERT to get the link id back.
      */
     if ($isNew) {
         $columnsArray = array($db->quoteName('url'), $db->quoteName('route'), $db->quoteName('title'), $db->quoteName('description'), $db->quoteName('indexdate'), $db->quoteName('published'), $db->quoteName('state'), $db->quoteName('access'), $db->quoteName('language'), $db->quoteName('type_id'), $db->quoteName('object'), $db->quoteName('publish_start_date'), $db->quoteName('publish_end_date'), $db->quoteName('start_date'), $db->quoteName('end_date'), $db->quoteName('list_price'), $db->quoteName('sale_price'));
         // Insert the link.
         $query->clear()->insert($db->quoteName('#__finder_links'))->columns($columnsArray)->values($db->quote($item->url) . ', ' . $db->quote($item->route) . ', ' . $db->quote($item->title) . ', ' . $db->quote($item->description) . ', ' . $query->currentTimestamp() . ', ' . '1, ' . (int) $item->state . ', ' . (int) $item->access . ', ' . $db->quote($item->language) . ', ' . (int) $item->type_id . ', ' . $db->quote(serialize($item)) . ', ' . $db->quote($item->publish_start_date) . ', ' . $db->quote($item->publish_end_date) . ', ' . $db->quote($item->start_date) . ', ' . $db->quote($item->end_date) . ', ' . (double) ($item->list_price ? $item->list_price : 0) . ', ' . (double) ($item->sale_price ? $item->sale_price : 0));
         $db->setQuery($query);
         $db->execute();
         // Get the link id.
         $linkId = (int) $db->insertid();
     } else {
         // Update the link.
         $query->clear()->update($db->quoteName('#__finder_links'))->set($db->quoteName('route') . ' = ' . $db->quote($item->route))->set($db->quoteName('title') . ' = ' . $db->quote($item->title))->set($db->quoteName('description') . ' = ' . $db->quote($item->description))->set($db->quoteName('indexdate') . ' = ' . $query->currentTimestamp())->set($db->quoteName('state') . ' = ' . (int) $item->state)->set($db->quoteName('access') . ' = ' . (int) $item->access)->set($db->quoteName('language') . ' = ' . $db->quote($item->language))->set($db->quoteName('type_id') . ' = ' . (int) $item->type_id)->set($db->quoteName('object') . ' = ' . $db->quote(serialize($item)))->set($db->quoteName('publish_start_date') . ' = ' . $db->quote($item->publish_start_date))->set($db->quoteName('publish_end_date') . ' = ' . $db->quote($item->publish_end_date))->set($db->quoteName('start_date') . ' = ' . $db->quote($item->start_date))->set($db->quoteName('end_date') . ' = ' . $db->quote($item->end_date))->set($db->quoteName('list_price') . ' = ' . (double) ($item->list_price ? $item->list_price : 0))->set($db->quoteName('sale_price') . ' = ' . (double) ($item->sale_price ? $item->sale_price : 0))->where('link_id = ' . (int) $linkId);
         $db->setQuery($query);
         $db->execute();
     }
     // Set up the variables we will need during processing.
     $count = 0;
     // Mark afterLinking in the profiler.
     static::$profiler ? static::$profiler->mark('afterLinking') : null;
     // Truncate the tokens tables.
     $db->truncateTable('#__finder_tokens');
     // Truncate the tokens aggregate table.
     $db->truncateTable('#__finder_tokens_aggregate');
     /*
      * Process the item's content. The items can customize their
      * processing instructions to define extra properties to process
      * or rearrange how properties are weighted.
      */
     foreach ($item->getInstructions() as $group => $properties) {
         // Iterate through the properties of the group.
         foreach ($properties as $property) {
             // Check if the property exists in the item.
             if (empty($item->{$property})) {
                 continue;
             }
             // Tokenize the property.
             if (is_array($item->{$property})) {
                 // Tokenize an array of content and add it to the database.
//.........这里部分代码省略.........

 /**
  * Tests the parse method
  *
  * @return  void
  *
  * @since   3.0
  */
 public function testParse()
 {
     $this->assertThat(FinderIndexerHelper::parse('Test string to parse with the txt parser', 'txt'), $this->stringContains('Test string to parse with the txt parser'), 'Tests that FinderIndexerHelper::parse() returns the string given with the txt parser.');
 }