PHP strip_punctuation函数代码示例

/**
 * nv_show_tags_list()
 *
 * @param string $q
 * @param integer $incomplete
 * @return
 */
function nv_show_tags_list($q = '', $incomplete = false)
{
    global $db_slave, $lang_module, $lang_global, $module_name, $module_data, $op, $module_file, $global_config, $module_info, $module_config, $nv_Request;
    $db_slave->sqlreset()->select('*')->from(NV_PREFIXLANG . '_' . $module_data . '_tags')->order('alias ASC');
    if ($incomplete === true) {
        $db_slave->where('description = \'\'');
    }
    if (!empty($q)) {
        $q = strip_punctuation($q);
        $db_slave->where('keywords LIKE :keywords');
    } else {
        $per_page = $nv_Request->get_int('per_page', 'cookie', $module_config[$module_name]['per_page']);
        $db_slave->order('alias ASC')->limit($per_page);
    }
    $sth = $db_slave->prepare($db_slave->sql());
    if (!empty($q)) {
        $sth->bindValue(':keywords', '%' . $q . '%', PDO::PARAM_STR);
    }
    $sth->execute();
    $xtpl = new XTemplate('tags_lists.tpl', NV_ROOTDIR . '/themes/' . $global_config['module_theme'] . '/modules/' . $module_file);
    $xtpl->assign('LANG', $lang_module);
    $xtpl->assign('GLANG', $lang_global);
    $xtpl->assign('MODULE_NAME', $module_name);
    $xtpl->assign('OP', $op);
    $xtpl->assign('SITEKEY', md5($global_config['sitekey'] . session_id()));
    $number = 0;
    while ($row = $sth->fetch()) {
        $row['number'] = ++$number;
        $row['link'] = NV_BASE_SITEURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=' . $module_info['alias']['tag'] . '/' . $row['alias'];
        $row['url_edit'] = NV_BASE_ADMINURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=' . $op . '&tid=' . $row['tid'] . ($incomplete === true ? '&incomplete=1' : '') . '#edit';
        $xtpl->assign('ROW', $row);
        if (empty($row['description']) and $incomplete === false) {
            $xtpl->parse('main.loop.incomplete');
        }
        $xtpl->parse('main.loop');
    }
    $sth->closeCursor();
    if (empty($q) and $number >= $module_config[$module_name]['per_page']) {
        $xtpl->parse('main.other');
    }
    $xtpl->parse('main');
    $contents = $xtpl->text('main');
    if (empty($contents)) {
        $contents = ' ';
    }
    return $contents;
}

/**
 * nv_show_tags_list()
 *
 * @param string $q
 * @param integer $incomplete
 * @return
 */
function nv_show_tags_list($q = '', $incomplete = false)
{
    global $db, $lang_module, $lang_global, $module_name, $module_data, $op, $module_file, $global_config, $module_info, $module_config, $nv_Request;
    $base_url = NV_BASE_ADMINURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=' . $op;
    $db->sqlreset()->select('*')->from(NV_PREFIXLANG . '_' . $module_data . '_tags')->order('alias ASC');
    if (!empty($q)) {
        $q = strip_punctuation($q);
        $db->where('keywords LIKE %' . $q . '%');
    }
    if ($incomplete === true) {
        $db->where('description = \'\'');
    }
    $num_items = $db->query($db->sql())->fetchColumn();
    $page = $nv_Request->get_int('page', 'post,get', 1);
    $per_page = 20;
    $db->select('*')->order('alias ASC')->limit($per_page)->offset(($page - 1) * $per_page);
    $result2 = $db->query($db->sql());
    $xtpl = new XTemplate('tags_lists.tpl', NV_ROOTDIR . '/themes/' . $global_config['module_theme'] . '/modules/' . $module_file);
    $xtpl->assign('LANG', $lang_module);
    $xtpl->assign('GLANG', $lang_global);
    $number = 0;
    while ($row = $result2->fetch()) {
        $row['number'] = ++$number;
        $row['link'] = NV_BASE_SITEURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=tag/' . $row['alias'];
        $row['url_edit'] = NV_BASE_ADMINURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=' . $op . '&did=' . $row['did'] . ($incomplete === true ? '&incomplete=1' : '') . '#edit';
        $xtpl->assign('ROW', $row);
        if (empty($row['description']) and $incomplete === false) {
            $xtpl->parse('main.loop.incomplete');
        }
        $xtpl->parse('main.loop');
    }
    if (empty($q) and $number >= 20) {
        $xtpl->parse('main.other');
    }
    $generate_page = nv_generate_page($base_url, $num_items, $per_page, $page);
    if (!empty($generate_page)) {
        $xtpl->assign('GENERATE_PAGE', $generate_page);
        $xtpl->parse('main.generate_page');
    }
    $xtpl->parse('main');
    $contents = $xtpl->text('main');
    if (empty($contents)) {
        $contents = ' ';
    }
    return $contents;
}

 */
if (!defined('NV_IS_MOD_NEWS')) {
    die('Stop!!!');
}
$alias = $nv_Request->get_title('alias', 'get');
$array_op = explode('/', $alias);
$alias = $array_op[0];
if (isset($array_op[1])) {
    if (sizeof($array_op) == 2 and preg_match('/^page\\-([0-9]+)$/', $array_op[1], $m)) {
        $page = intval($m[1]);
    } else {
        $alias = '';
    }
}
$page_title = trim(str_replace('-', ' ', $alias));
if (!empty($page_title) and $page_title == strip_punctuation($page_title)) {
    $stmt = $db->prepare('SELECT tid, image, description, keywords FROM ' . NV_PREFIXLANG . '_' . $module_data . '_tags WHERE alias= :alias');
    $stmt->bindParam(':alias', $alias, PDO::PARAM_STR);
    $stmt->execute();
    list($tid, $image_tag, $description, $key_words) = $stmt->fetch(3);
    if ($tid > 0) {
        $base_url = NV_BASE_SITEURL . 'index.php?' . NV_LANG_VARIABLE . '=' . NV_LANG_DATA . '&' . NV_NAME_VARIABLE . '=' . $module_name . '&' . NV_OP_VARIABLE . '=tag/' . $alias;
        if ($page > 1) {
            $page_title .= ' ' . NV_TITLEBAR_DEFIS . ' ' . $lang_global['page'] . ' ' . $page;
        }
        $array_mod_title[] = array('catid' => 0, 'title' => $page_title, 'link' => $base_url);
        $item_array = array();
        $end_publtime = 0;
        $show_no_image = $module_config[$module_name]['show_no_image'];
        $db->sqlreset()->select('COUNT(*)')->from(NV_PREFIXLANG . '_' . $module_data . '_rows')->where('status=1 AND id IN (SELECT id FROM ' . NV_PREFIXLANG . '_' . $module_data . '_tags_id WHERE tid=' . $tid . ')');
        $num_items = $db->query($db->sql())->fetchColumn();

 $row['socialbutton'] = $nv_Request->get_int('socialbutton', 'post', 0);
 $row['layout_func'] = $nv_Request->get_title('layout_func', 'post', '');
 $row['gid'] = $nv_Request->get_int('gid', 'post', 0);
 $_groups_post = $nv_Request->get_array('activecomm', 'post', array());
 $row['activecomm'] = !empty($_groups_post) ? implode(',', nv_groups_post(array_intersect($_groups_post, array_keys($groups_list)))) : '';
 if (empty($row['title'])) {
     $error = $lang_module['empty_title'];
 } elseif (strip_tags($row['bodytext']) == '') {
     $error = $lang_module['empty_bodytext'];
 } elseif (empty($row['layout_func']) or in_array('layout.' . $row['layout_func'] . '.tpl', $layout_array)) {
     $row['alias'] = empty($row['alias']) ? change_alias($row['title']) : change_alias($row['alias']);
     if (empty($row['keywords'])) {
         $row['keywords'] = nv_get_keywords($row['title']);
         if (empty($row['keywords'])) {
             $row['keywords'] = nv_unhtmlspecialchars($row['keywords']);
             $row['keywords'] = strip_punctuation($row['keywords']);
             $row['keywords'] = trim($row['keywords']);
             $row['keywords'] = nv_strtolower($row['keywords']);
             $row['keywords'] = preg_replace('/[ ]+/', ',', $row['keywords']);
         }
     }
     if ($id) {
         $_sql = 'UPDATE ' . NV_PREFIXLANG . '_' . $module_data . ' SET title = :title, alias = :alias, image = :image, imagealt = :imagealt, imageposition = :imageposition, description = :description, bodytext = :bodytext, keywords = :keywords, socialbutton = :socialbutton, activecomm = :activecomm, layout_func = :layout_func, gid = :gid, admin_id = :admin_id, edit_time = ' . NV_CURRENTTIME . ' WHERE id =' . $id;
         $publtime = $row['add_time'];
     } else {
         if ($page_config['news_first']) {
             $weight = 1;
         } else {
             $weight = $db->query("SELECT MAX(weight) FROM " . NV_PREFIXLANG . "_" . $module_data)->fetchColumn();
             $weight = intval($weight) + 1;
         }

function doctermTags($req)
{
    global $CONFIG;
    set_time_limit(0);
    //this avoids timeouts
    require_once $CONFIG->path . "mod/profile_manager/views/default/profile_manager/members/config.php";
    require_once $CONFIG->path . "mod/profile_manager/views/default/profile_manager/members/classes.php";
    require_once $CONFIG->path . "mod/profile_manager/views/default/profile_manager/members/stemming.php";
    $outputfile = $CONFIG->path . "mod/profile_manager/views/default/profile_manager/members/output.log";
    file_put_contents($outputfile, "Starting creating TAGS Doc-term matrix...\n", FILE_APPEND);
    $guids = unserialize(file_get_contents($IOdir . "guids"));
    $lr_array = unserialize(file_get_contents($IOdir . "lr"));
    $tags_dt = array();
    $stop_words = set_stop_words_tags($CONFIG->path . "mod/profile_manager/views/default/profile_manager/members/stop_words_eng.txt");
    //create an array containing "stop words", in order to eliminate them from the text
    if ($req["dt_useold"] == "true" && file_exists($IOdir . "old_lr") && file_exists($IOdir . "old_tags_dt_raw")) {
        $old_lr = unserialize(file_get_contents($IOdir . "old_lr"));
        $old_tags_dt_raw = unserialize(file_get_contents($IOdir . "old_tags_dt_raw"));
        //we need the raw version of the doc-term matrix, before applying IDF or synonyms
    }
    //create an array containing all the tags of each document
    foreach ($guids as $guid) {
        if (!empty($lr_array[$guid]->tags)) {
            if ($req["dt_useold"] == "true" && isset($old_lr[$guid]->tags) && $lr_array[$guid]->tags == $old_lr[$guid]->tags && isset($old_tags_dt_raw[$guid])) {
                $tags_dt[$guid] = $old_tags_dt_raw[$guid];
                continue;
            }
            //find all the tags for the current resource
            $tags = array();
            foreach ($lr_array[$guid]->tags as $sentence) {
                if ($sentence == "") {
                    continue;
                }
                $sentence = strip_punctuation($sentence);
                //strip punctuation
                $sentence_clean = str_replace($stop_words, " ", $sentence);
                //eliminate stop words
                $tags_sentence = explode(" ", $sentence_clean);
                $tags = array_merge($tags, $tags_sentence);
            }
            //stem each tag
            foreach ($tags as $num => $element) {
                $tags[$num] = PorterStemmer::Stem(strtolower(trim($element)));
                //stem elements
            }
            $tags = array_filter(array_unique($tags));
            //delete duplicates and empty elements
            //create the entry for the current document in the doc-term tags matrix
            foreach ($tags as $tag) {
                $tags_dt[$guid][$tag] = 1;
            }
        }
    }
    file_put_contents($IOdir . "tags_dt_raw", serialize($tags_dt));
    //it saves the raw version of the doc-term matrix
    if (PHP_OS == "Linux") {
        chmod($IOdir . "tags_dt_raw", 0666);
    }
    //set rw permissions for everybody for this file
    if ($enable_synonyms) {
        foreach ($keys as $num => $key) {
            if (!isset($keys[$num])) {
                continue;
            }
            //since there is an unset on this array into the foreach, we have to check if the present key is still available or not
            foreach ($keys as $num2 => $key2) {
                if (!isset($keys[$num2])) {
                    continue;
                }
                //since there is an unset on this array into the foreach, we have to check if the present key is still available or not
                if ($key != $key2 && check_synonyms($key, $key2)) {
                    file_put_contents($outputfile, "\n{$key} and {$key2} are synonyms\n", FILE_APPEND);
                    foreach ($tags_dt as $guid => $element) {
                        if (isset($tags_dt[$guid][$key2])) {
                            unset($tags_dt[$guid][$key2]);
                            $tags_dt[$guid][$key] = 1;
                        }
                    }
                    unset($keys[$num2]);
                }
            }
        }
    }
    file_put_contents($IOdir . "tags_dt", serialize($tags_dt));
    if (PHP_OS == "Linux" && posix_getuid() == fileowner($IOdir . 'tags_dt')) {
        chmod($IOdir . 'tags_dt', 0666);
    }
    //set rw permissions for everybody for this file
    file_put_contents($outputfile, "TAGS Doc-term matrix created\n\n", FILE_APPEND);
    return "OK";
}

 }
 if (!empty($rowdetail['image']) && !nv_is_url($rowdetail['image'])) {
     $imagesize = @getimagesize(NV_UPLOADS_REAL_DIR . '/' . $module_upload . '/' . $rowdetail['image']);
     $rowdetail['image'] = NV_BASE_SITEURL . NV_UPLOADS_DIR . '/' . $module_upload . '/' . $rowdetail['image'];
     $rowdetail['imageWidth'] = $imagesize[0] > 500 ? 500 : $imagesize[0];
 }
 $rowdetail['add_time'] = nv_date('H:i T l, d/m/Y', $rowdetail['add_time']);
 $rowdetail['edit_time'] = nv_date('H:i T l, d/m/Y', $rowdetail['edit_time']);
 $module_info['layout_funcs'][$op_file] = !empty($rowdetail['layout_func']) ? $rowdetail['layout_func'] : $module_info['layout_funcs'][$op_file];
 if (!empty($rowdetail['keywords'])) {
     $key_words = $rowdetail['keywords'];
 } else {
     $key_words = nv_get_keywords($rowdetail['bodytext']);
     if (empty($key_words)) {
         $key_words = nv_unhtmlspecialchars($rowdetail['title']);
         $key_words = strip_punctuation($key_words);
         $key_words = trim($key_words);
         $key_words = nv_strtolower($key_words);
         $key_words = preg_replace('/[ ]+/', ',', $key_words);
     }
 }
 $page_title = $mod_title = $rowdetail['title'];
 $description = $rowdetail['description'];
 $id_profile_googleplus = $rowdetail['gid'];
 // Hiển thị các bài liên quan mới nhất.
 $other_links = array();
 $related_articles = intval($page_config['related_articles']);
 if ($related_articles) {
     $db_slave->sqlreset()->select('*')->from(NV_PREFIXLANG . '_' . $module_data)->where('status=1 AND id !=' . $id)->order('weight ASC')->limit($related_articles);
     $result = $db_slave->query($db_slave->sql());
     while ($_other = $result->fetch()) {

 $result = get_web_page($url);
 if ($result['errno'] != 0) {
     echo " error: bad url, timeout, redirect loop ";
 }
 if ($result['http_code'] != 200) {
     echo "error: no page, no permissions, no service";
 }
 $page = $result['content'];
 $content_type = $result['content_type'];
 $mime = $result['mime'];
 $charset = $result['charset'];
 echo "This site is encoded with" . " " . "<b>" . $charset . "</b>" . " " . "format" . "<br>";
 $utf8_text = $result['utf8_text'];
 $text = strip_html_tags($page);
 $utf8_text = html_entity_decode($text, ENT_QUOTES, "utf-8");
 $utf8_text = strip_punctuation($utf8_text);
 $utf8_text = strip_symbols($utf8_text);
 $utf8_text = strip_numbers($utf8_text);
 mb_regex_encoding("utf-8");
 $words = mb_split(' +', $utf8_text);
 foreach ($words as $key => $word) {
     $words[$key] = PorterStemmer::Stem($word, true);
 }
 $stopWords = mb_split('[ \\n]+', mb_strtolower($words[$key], 'utf-8'));
 foreach ($stopWords as $key => $word) {
     $stopWords[$key] = PorterStemmer::Stem($word, true);
 }
 $words = array_diff($words, $stopWords);
 $keywordCounts = array_count_values($words);
 arsort($keywordCounts, SORT_NUMERIC);
 $uniqueKeywords = array_keys($keywordCounts);

$key = filter_text_input('search_query', 'get', '', 0, NV_MAX_SEARCH_LENGTH);
$len_key = 0;
$logic = filter_text_input('logic', 'get', 'OR');
if ($logic != 'AND') {
    $logic = 'OR';
}
$checkss = filter_text_input('search_ss', 'get', '', 1, 32);
$ss = md5($client_info['session_id'] . $global_config['sitekey']);
if (!preg_match("/^[a-z0-9]{32}\$/", $checkss) or $checkss != $ss) {
    $key = "";
} else {
    $key = nv_unhtmlspecialchars($key);
    if ($logic == 'OR') {
        $key = preg_replace(array("/^([\\S]{1})\\s/uis", "/\\s([\\S]{1})\\s/uis", "/\\s([\\S]{1})\$/uis"), " ", $key);
    }
    $key = strip_punctuation($key);
    $key = trim($key);
    $len_key = nv_strlen($key);
    $key = nv_htmlspecialchars($key);
}
if ($len_key < NV_MIN_SEARCH_LENGTH) {
    die('&nbsp;&nbsp;');
}
$pages = $nv_Request->get_int('page', 'get', 0);
$contents = "";
$ss = md5($client_info['session_id'] . $global_config['sitekey']);
foreach ($mods as $m_name => $m_values) {
    $all_page = 0;
    $result_array = array();
    $dbkeyword = $db->dblikeescape($key);
    include NV_ROOTDIR . "/modules/" . $m_values['module_file'] . "/search.php";

function Index($page_id)
{
    global $db;
    $needles = array("/\\ba\\b/i", "/\\bable\\b/i", "/\\babout\\b/i", "/\\bacross\\b/i", "/\\bafter\\b/i", "/\\ball\\b/i", "/\\balmost\\b/i", "/\\balso\\b/i", "/\\bam\\b/i", "/\\bamong\\b/i", "/\\ban\\b/i", "/\\band\\b/i", "/\\bany\\b/i", "/\\bare\\b/i", "/\\bas\\b/i", "/\\bat\\b/i", "/\\bbe\\b/i", "/\\bbecause\\b/i", "/\\bbeen\\b/i", "/\\bbut\\b/i", "/\\bby\\b/i", "/\\bcan\\b/i", "/\\bcannot\\b/i", "/\\bcould\\b/i", "/\\bdear\\b/i", "/\\bdid\\b/i", "/\\bdo\\b/i", "/\\bdoes\\b/i", "/\\beither\\b/i", "/\\belse\\b/i", "/\\bever\\b/i", "/\\bevery\\b/i", "/\\bfor\\b/i", "/\\bfrom\\b/i", "/\\bget\\b/i", "/\\bgot\\b/i", "/\\bhad\\b/i", "/\\bhas\\b/i", "/\\bhave\\b/i", "/\\bhe\\b/i", "/\\bher\\b/i", "/\\bhers\\b/i", "/\\bhim\\b/i", "/\\bhis\\b/i", "/\\bhow\\b/i", "/\\bhowever\\b/i", "/\\bi\\b/i", "/\\bif\\b/i", "/\\bin\\b/i", "/\\binto\\b/i", "/\\bis\\b/i", "/\\bit\\b/i", "/\\bits\\b/i", "/\\bjust\\b/i", "/\\bleast\\b/i", "/\\blet\\b/i", "/\\blike\\b/i", "/\\blikely\\b/i", "/\\bmay\\b/i", "/\\bme\\b/i", "/\\bmight\\b/i", "/\\bmost\\b/i", "/\\bmust\\b/i", "/\\bmy\\b/i", "/\\bneither\\b/i", "/\\bno\\b/i", "/\\bnor\\b/i", "/\\bnot\\b/i", "/\\bof\\b/i", "/\\boff\\b/i", "/\\boften\\b/i", "/\\bon\\b/i", "/\\bonly\\b/i", "/\\bor\\b/i", "/\\bother\\b/i", "/\\bour\\b/i", "/\\bown\\b/i", "/\\brather\\b/i", "/\\bsaid\\b/i", "/\\bsay\\b/i", "/\\bsays\\b/i", "/\\bshe\\b/i", "/\\bshould\\b/i", "/\\bsince\\b/i", "/\\bso\\b/i", "/\\bsome\\b/i", "/\\bthan\\b/i", "/\\bthat\\b/i", "/\\bthe\\b/i", "/\\btheir\\b/i", "/\\bthem\\b/i", "/\\bthen\\b/i", "/\\bthere\\b/i", "/\\bthese\\b/i", "/\\bthey\\b/i", "/\\bthis\\b/i", "/\\btis\\b/i", "/\\bto\\b/i", "/\\btoo\\b/i", "/\\btwas\\b/i", "/\\bus\\b/i", "/\\bwants\\b/i", "/\\bwas\\b/i", "/\\bwe\\b/i", "/\\bwere\\b/i", "/\\bwhat\\b/i", "/\\bwhen\\b/i", "/\\bwhere\\b/i", "/\\bwhich\\b/i", "/\\bwhile\\b/i", "/\\bwho\\b/i", "/\\bwhom\\b/i", "/\\bwhy\\b/i", "/\\bwill\\b/i", "/\\bwith\\b/i", "/\\bwould\\b/i", "/\\byet\\b/i", "/\\byou\\b/i", "/\\byour\\b/i", "/\\bain't\\b/i", "/\\baren't\\b/i", "/\\bcan't\\b/i", "/\\bcould've\\b/i", "/\\bcouldn't\\b/i", "/\\bdidn't\\b/i", "/\\bdoesn't\\b/i", "/\\bdon't\\b/i", "/\\bhasn't\\b/i", "/\\bhe'd\\b/i", "/\\bhe'll\\b/i", "/\\bhe's\\b/i", "/\\bhow'd\\b/i", "/\\bhow'll\\b/i", "/\\bhow's\\b/i", "/\\bi'd\\b/i", "/\\bi'll\\b/i", "/\\bi'm\\b/i", "/\\bi've\\b/i", "/\\bisn't\\b/i", "/\\bit's\\b/i", "/\\bmight've\\b/i", "/\\bmightn't\\b/i", "/\\bmust've\\b/i", "/\\bmustn't\\b/i", "/\\bshan't\\b/i", "/\\bshe'd\\b/i", "/\\bshe'll\\b/i", "/\\bshe's\\b/i", "/\\bshould've\\b/i", "/\\bshouldn't\\b/i", "/\\bthat'll\\b/i", "/\\bthat's\\b/i", "/\\bthere's\\b/i", "/\\bthey'd\\b/i", "/\\bthey'll\\b/i", "/\\bthey're\\b/i", "/\\bthey've\\b/i", "/\\bwasn't\\b/i", "/\\bwe'd\\b/i", "/\\bwe'll\\b/i", "/\\bwe're\\b/i", "/\\bweren't\\b/i", "/\\bwhat'd\\b/i", "/\\bwhat's\\b/i", "/\\bwhen'd\\b/i", "/\\bwhen'll\\b/i", "/\\bwhen's\\b/i", "/\\bwhere'd\\b/i", "/\\bwhere'll\\b/i", "/\\bwhere's\\b/i", "/\\bwho'd\\b/i", "/\\bwho'll\\b/i", "/\\bwho's\\b/i", "/\\bwhy'd\\b/i", "/\\bwhy'll\\b/i", "/\\bwhy's\\b/i", "/\\bwon't\\b/i", "/\\bwould've\\b/i", "/\\bwouldn't\\b/i", "/\\byou'd\\b/i", "/\\byou'll\\b/i", "/\\byou're\\b/i", "/\\byou've\\b/i");
    $query = $db->query("SELECT contents FROM page WHERE id ={$page_id}");
    $page_contents = $query->fetchAll(PDO::FETCH_ASSOC);
    foreach ($page_contents as $row) {
        $utf8_text = strip_html_tags($row['contents']);
        $utf8_text = html_entity_decode($utf8_text, ENT_QUOTES, "UTF-8");
        $text = strip_punctuation($utf8_text);
        $text = strip_symbols($text);
        $text = strip_numbers($text);
        $text = mb_strtolower($text, "utf-8");
        $words = explode(' ', $text);
        $keywordCounts = array_count_values($words);
        arsort($keywordCounts, SORT_NUMERIC);
        $uniqueKeywords = array_keys($keywordCounts);
        $uniqueKeywords = preg_replace($needles, "", $uniqueKeywords);
        //Remove stop words
        // print_r($uniqueKeywords);
        for ($i = 0; $i < count($uniqueKeywords); $i++) {
            if ($uniqueKeywords[$i] != "") {
                $cur_word = addslashes(strtolower($uniqueKeywords[$i]));
                $query = $db->query("SELECT word_id FROM word WHERE word_word='{$cur_word}'");
                $row = $query->fetchAll();
                if ($row) {
                    if ($row[0]['word_id']) {
                        $word_id = $row[0]['word_id'];
                    }
                } else {
                    $query = $db->query("INSERT INTO word (word_word) VALUES ('{$cur_word}')");
                    $word_id = $db->lastInsertId();
                }
                $db->query("INSERT INTO occurrence (word_id,page_id) VALUES ({$word_id},{$page_id})");
            }
        }
    }
    return "Index";
}

    die('Stop!!!');
}
$page_title = $lang_module['keywordRank'];
$contents = '';
$xtpl = new XTemplate('keywordRank.tpl', NV_ROOTDIR . '/themes/' . $global_config['module_theme'] . '/modules/' . $module_file);
$xtpl->assign('NV_NAME_VARIABLE', NV_NAME_VARIABLE);
$xtpl->assign('NV_OP_VARIABLE', NV_OP_VARIABLE);
$xtpl->assign('NV_BASE_SITEURL', NV_BASE_SITEURL);
$xtpl->assign('LANG', $lang_module);
$xtpl->assign('TITLE', sprintf($lang_module['keywordFormTitle'], NV_SERVER_NAME));
if ($nv_Request->isset_request('i', 'get')) {
    $i = $nv_Request->get_string('i', 'get');
    if ($i == 'process') {
        $keyword = nv_substr($nv_Request->get_title('k', 'get', '', 0), 0, 60);
        $keyword = nv_unhtmlspecialchars($keyword);
        $keyword = strip_punctuation($keyword);
        $keyword = trim($keyword);
        $len_key = nv_strlen($keyword);
        //$keyword = nv_htmlspecialchars( $keyword );
        $lang = $nv_Request->get_title('l', 'get', '');
        $accuracy = $nv_Request->get_title('a', 'get', '');
        if ($len_key < 3 or $len_key > 60) {
            $xtpl->assign('ERROR', $lang_module['keywordInfo']);
            $xtpl->parse('process.error');
        } else {
            $myDomain = NV_SERVER_NAME;
            // $myDomain = 'nukeviet.vn';
            $from = 'google';
            $tempFile = md5($keyword . $lang . $accuracy . $from . $myDomain);
            $tempFile = NV_ROOTDIR . '/' . NV_TEMP_DIR . '/' . NV_TEMPNAM_PREFIX . $tempFile;
            if (file_exists($tempFile) and @filemtime($tempFile) > NV_CURRENTTIME - 600) {

/**
 * nv_get_keywords()
 *
 * @param string $content
 * @return
 */
function nv_get_keywords($content, $keyword_limit = 20)
{
    $content = strip_tags($content);
    $content = nv_unhtmlspecialchars($content);
    $content = strip_punctuation($content);
    $content = trim($content);
    $content = nv_strtolower($content);
    $content = ' ' . $content . ' ';
    $keywords_return = array();
    $memoryLimitMB = (int) ini_get('memory_limit');
    if ($memoryLimitMB > 60 and file_exists(NV_ROOTDIR . '/includes/keywords/' . NV_LANG_DATA . '.php')) {
        require NV_ROOTDIR . '/includes/keywords/' . NV_LANG_DATA . '.php';
        $content_array = explode(' ', $content);
        $a = 0;
        $b = sizeof($content_array);
        for ($i = 0; $i < $b - 3; ++$i) {
            $key3 = $content_array[$i] . ' ' . $content_array[$i + 1] . ' ' . $content_array[$i + 2];
            $key2 = $content_array[$i] . ' ' . $content_array[$i + 1];
            if (array_search($key3, $array_keywords_3)) {
                $keywords_return[] = $key3;
                $i = $i + 2;
            } elseif (array_search($key2, $array_keywords_2)) {
                $keywords_return[] = $key2;
                $i = $i + 1;
            }
            $keywords_return = array_unique($keywords_return);
            if (sizeof($keywords_return) > $keyword_limit) {
                break;
            }
        }
    } else {
        $pattern_word = array();
        if (NV_SITEWORDS_MIN_3WORDS_LENGTH > 0 and NV_SITEWORDS_MIN_3WORDS_PHRASE_OCCUR > 0) {
            $pattern_word[] = "/[\\s]+([\\S]{" . NV_SITEWORDS_MIN_3WORDS_LENGTH . ",}\\s[\\S]{" . NV_SITEWORDS_MIN_3WORDS_LENGTH . ",}\\s[\\S]{" . NV_SITEWORDS_MIN_3WORDS_LENGTH . ",})[\\s]+/uis";
        }
        if (NV_SITEWORDS_MIN_2WORDS_LENGTH > 0 and NV_SITEWORDS_MIN_2WORDS_PHRASE_OCCUR > 0) {
            $pattern_word[] = "/[\\s]+([\\S]{" . NV_SITEWORDS_MIN_2WORDS_LENGTH . ",}\\s[\\S]{" . NV_SITEWORDS_MIN_2WORDS_LENGTH . ",})[\\s]+/uis";
        }
        if (NV_SITEWORDS_MIN_WORD_LENGTH > 0 and NV_SITEWORDS_MIN_WORD_OCCUR > 0) {
            $pattern_word[] = "/[\\s]+([\\S]{" . NV_SITEWORDS_MIN_WORD_LENGTH . ",})[\\s]+/uis";
        }
        if (empty($pattern_word)) {
            return '';
        }
        $lenght = 0;
        $max_strlen = min(NV_SITEWORDS_MAX_STRLEN, 300);
        foreach ($pattern_word as $pattern) {
            while (preg_match($pattern, $content, $matches)) {
                $keywords_return[] = $matches[1];
                $lenght += nv_strlen($matches[1]);
                $content = preg_replace("/[\\s]+(" . preg_quote($matches[1]) . ")[\\s]+/uis", ' ', $content);
                if ($lenght >= $max_strlen) {
                    break;
                }
            }
            if ($lenght >= $max_strlen) {
                break;
            }
        }
        $keywords_return = array_unique($keywords_return);
    }
    return implode(',', $keywords_return);
}

 function parseContentBodyItems($original_content, $title)
 {
     $original_content = clean_word($original_content);
     $original_content = $original_content;
     $rem = false;
     $original_content = str_replace($rem, ' ', $original_content);
     $rem = ' ';
     $original_content = str_ireplace($rem, ' ', $original_content);
     $original_content = str_ireplace('class=mcevisualaid>', ' ', $original_content);
     //  $original_content = html_entity_decode($original_content);
     $site_url = site_url();
     $original_content = str_ireplace($site_url, '{SITEURL}', $original_content);
     $content_item = $original_content;
     $possible_filename = CI::model('core')->url_title($title, 'dash', true);
     $possible_filename = string_cyr2lat($possible_filename);
     $possible_filename = $possible_filename . '-' . date("Ymdhis") . rand(1, 99);
     $possible_filename = str_ireplace('&', '_', $possible_filename);
     $possible_filename = str_ireplace(';', '_', $possible_filename);
     $possible_filename = str_ireplace('`', '_', $possible_filename);
     $possible_filename = str_ireplace('"', '_', $possible_filename);
     $possible_filename = str_ireplace("'", '_', $possible_filename);
     $possible_filename = str_ireplace("%", '_', $possible_filename);
     $possible_filename = str_ireplace("*", '_', $possible_filename);
     $possible_filename = str_ireplace("#", '_', $possible_filename);
     $possible_filename = str_ireplace("@", '_', $possible_filename);
     $possible_filename = str_ireplace("!", '_', $possible_filename);
     $possible_filename = str_ireplace('$', '_', $possible_filename);
     $possible_filename = str_ireplace('/', '_', $possible_filename);
     $possible_filename = str_ireplace('\\', '_', $possible_filename);
     $possible_filename = str_ireplace('[', '_', $possible_filename);
     $possible_filename = str_ireplace(']', '_', $possible_filename);
     $possible_filename = str_ireplace('â€â"¢', '_', $possible_filename);
     if (strval($original_content) == '') {
         return false;
     }
     $input = $content_item;
     $regexp = "<img\\s[^>]*src=(\"??)([^\" >]*?)\\1[^>]*";
     $images = array();
     if (preg_match_all("/{$regexp}/siU", $input, $matches, PREG_SET_ORDER)) {
         foreach ($matches as $match) {
             # $match[2] = link address
             # $match[3] = link text
             $images[] = $match[2];
         }
     }
     $dir = MEDIAFILES . 'downloaded/';
     if (is_dir($dir) == false) {
         @mkdir($dir);
     }
     @touch($dir . 'index.html');
     //mkdir
     $media_url = MEDIA_URL;
     if (!empty($images)) {
         foreach ($images as $image) {
             if (stristr($image, '.jpg') == true or stristr($image, '.png') == true or stristr($image, '.gif') == true or stristr($image, '.bmp') == true or stristr($image, '.jpeg') == true) {
                 $orig_image = $image;
                 if (stristr($image, '{MEDIAURL}' == false)) {
                     if (stristr($image, $media_url) == true) {
                     } else {
                         if (CI::model('core')->url_IsFile($image) == true) {
                             $to_get = $image;
                         } else {
                             $image = 'http://maksoft.net/' . $image;
                             if (CI::model('core')->url_IsFile($image) == true) {
                                 $to_get = $image;
                             }
                         }
                         if (CI::model('core')->url_IsFile($image) == true) {
                             if (stristr($image, $media_url) == false) {
                                 //print 'file: ' . $image;
                                 $parts = explode('/', $image);
                                 $currentFile = $parts[count($parts) - 1];
                                 $orig_file = $currentFile;
                                 $ext = substr($image, strrpos($image, '.') + 1);
                                 //$to_save =
                                 //exit($ext);
                                 $orig_file_clean = strip_punctuation($orig_file);
                                 $orig_file_clean = str_replace('.', '_', $orig_file_clean);
                                 $orig_file_clean = str_replace('.', '_', $orig_file_clean);
                                 $orig_file_clean = str_replace('.', '_', $orig_file_clean);
                                 $orig_file_clean = str_replace('=', '_', $orig_file_clean);
                                 $orig_file_clean = str_replace('?', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('&', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace(';', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('`', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('"', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("'", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("%", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("*", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("#", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("@", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace("!", '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('$', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('/', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('\\', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('[', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace(']', '_', $orig_file_clean);
                                 $orig_file_clean = str_ireplace('â€â"¢', '_', $orig_file_clean);
                                 $orig_file_clean = $orig_file_clean . '.' . $ext;
                                 if (is_file($dir . $possible_filename) == false) {
//.........这里部分代码省略.........

 $search['key'] = filter_text_input('q', 'get', '', 0, NV_MAX_SEARCH_LENGTH);
 $search['logic'] = $nv_Request->get_int('l', 'get', $search['logic']);
 $search['mod'] = filter_text_input('m', 'get', 'all', $search['mod']);
 $search['page'] = $nv_Request->get_int('page', 'get', 0);
 if ($search['logic'] != 1) {
     $search['logic'] = 0;
 }
 if (!isset($array_modul[$search['mod']])) {
     $search['mod'] = "all";
 }
 if (!empty($search['key'])) {
     $search['key'] = nv_unhtmlspecialchars($search['key']);
     if (!$search['logic']) {
         $search['key'] = preg_replace(array("/^([\\S]{1})\\s/uis", "/\\s([\\S]{1})\\s/uis", "/\\s([\\S]{1})\$/uis"), " ", $search['key']);
     }
     $search['key'] = strip_punctuation($search['key']);
     $search['key'] = trim($search['key']);
     $search['len_key'] = nv_strlen($search['key']);
     $search['key'] = nv_htmlspecialchars($search['key']);
 }
 if ($search['len_key'] < NV_MIN_SEARCH_LENGTH) {
     $search['is_error'] = true;
     $search['errorInfo'] = sprintf($lang_module['searchQueryError'], NV_MIN_SEARCH_LENGTH);
 } else {
     if (!empty($search['mod']) and isset($array_modul[$search['mod']])) {
         $mods = array($search['mod'] => $array_modul[$search['mod']]);
         $limit = 10;
         $is_generate_page = true;
     } else {
         $mods = $array_modul;
         $limit = 3;

 public static function parseExcerpt($keywords, $text)
 {
     $text = trim($text);
     $text = preg_replace("/\n/", '', $text);
     $text = preg_replace("/[\\s]{2,}/m", ' ', $text);
     // remove punctuation for highlighting
     require_once EXTENSIONS . '/search_index/lib/strip_punctuation.php';
     $keywords = strip_punctuation($keywords);
     $string_length = Symphony::Configuration()->get('excerpt-length', 'search_index') ? Symphony::Configuration()->get('excerpt-length', 'search_index') : 200;
     $between_start = $string_length / 2;
     $between_end = $string_length / 2;
     $elipsis = '...';
     // Extract positive keywords and phrases
     preg_match_all('/ ("([^"]+)"|(?!OR)([^" ]+))/', ' ' . $keywords, $matches);
     $keywords = array_merge($matches[2], $matches[3]);
     // don't highlight short words
     foreach ($keywords as $i => $keyword) {
         if (self::strlen($keyword) < 3) {
             unset($keywords[$i]);
         }
     }
     // Prepare text
     $text = ' ' . strip_tags(str_replace(array('<', '>'), array(' <', '> '), $text)) . ' ';
     // no idea what this next line actually does, nothing is harmed if it's simply commented out...
     array_walk($keywords, 'SearchIndex::_parseExcerptReplace');
     $workkeys = $keywords;
     // Extract a fragment per keyword for at most 4 keywords.
     // First we collect ranges of text around each keyword, starting/ending
     // at spaces.
     // If the sum of all fragments is too short, we look for second occurrences.
     $ranges = array();
     $included = array();
     $length = 0;
     while ($length < $string_length && count($workkeys)) {
         foreach ($workkeys as $k => $key) {
             if (self::strlen($key) == 0) {
                 unset($workkeys[$k]);
                 unset($keywords[$k]);
                 continue;
             }
             if ($length >= $string_length) {
                 break;
             }
             // Remember occurrence of key so we can skip over it if more occurrences
             // are desired.
             if (!isset($included[$key])) {
                 $included[$key] = 0;
             }
             // Locate a keyword (position $p), then locate a space in front (position
             // $q) and behind it (position $s)
             if (preg_match('/' . $boundary . $key . $boundary . '/iu', $text, $match, PREG_OFFSET_CAPTURE, $included[$key])) {
                 $p = $match[0][1];
                 if (($q = self::strpos($text, ' ', max(0, $p - $between_start))) !== FALSE) {
                     $end = self::substr($text, $p, $between_end);
                     if (($s = self::strrpos($end, ' ')) !== FALSE) {
                         $ranges[$q] = $p + $s;
                         $length += $p + $s - $q;
                         $included[$key] = $p + 1;
                     } else {
                         unset($workkeys[$k]);
                     }
                 } else {
                     unset($workkeys[$k]);
                 }
             } else {
                 unset($workkeys[$k]);
             }
         }
     }
     // If we didn't find anything, return the beginning.
     if (count($ranges) == 0) {
         if (self::strlen($text) > $string_length) {
             $text = self::substr($text, 0, $string_length) . $elipsis;
         }
         $text = General::sanitize($text);
         $text = preg_replace('/\\.\\.\\./', '&#8230;', $text);
         return '<p>' . $text . '</p>';
     }
     // Sort the text ranges by starting position.
     ksort($ranges);
     // Now we collapse overlapping text ranges into one. The sorting makes it O(n).
     $newranges = array();
     foreach ($ranges as $from2 => $to2) {
         if (!isset($from1)) {
             $from1 = $from2;
             $to1 = $to2;
             continue;
         }
         if ($from2 <= $to1) {
             $to1 = max($to1, $to2);
         } else {
             $newranges[$from1] = $to1;
             $from1 = $from2;
             $to1 = $to2;
         }
     }
     $newranges[$from1] = $to1;
     // Fetch text
     $out = array();
     foreach ($newranges as $from => $to) {
//.........这里部分代码省略.........

function doctermTags($lr_array)
{
    require_once 'stemming.php';
    global $IOdir, $enable_synonyms, $IndexingClassificationPath, $dt_new_indexing_required, $guids;
    echo 'Starting creating TAGS Doc-term matrix...' . "\n";
    $tags_dt = array();
    $stop_words = set_stop_words_tags($IndexingClassificationPath . "stop_words_eng.txt");
    //create an array containing "stop words", in order to eliminate them from the text
    if ($dt_new_indexing_required == 0 && file_exists($IOdir . "old_lr") && file_exists($IOdir . "old_tags_dt_raw")) {
        $old_lr = unserialize(file_get_contents($IOdir . "old_lr"));
        $old_tags_dt_raw = unserialize(file_get_contents($IOdir . "old_tags_dt_raw"));
    }
    //create an array containing all the tags of each document
    foreach ($guids as $guid) {
        if (!empty($lr_array[$guid]->tags)) {
            if ($dt_new_indexing_required == 0 && isset($old_lr[$guid]->tags) && $lr_array[$guid]->tags == $old_lr[$guid]->tags && isset($old_tags_dt_raw[$guid])) {
                $tags_dt[$guid] = $old_tags_dt_raw[$guid];
                continue;
            }
            //find all the tags for the current resource
            $tags = array();
            foreach ($lr_array[$guid]->tags as $sentence) {
                if ($sentence == "") {
                    continue;
                }
                $sentence = strip_punctuation($sentence);
                //strip punctuation
                $sentence_clean = str_replace($stop_words, " ", $sentence);
                //eliminate stop words
                $tags_sentence = explode(" ", $sentence_clean);
                $tags = array_merge($tags, $tags_sentence);
            }
            //stem each tag
            foreach ($tags as $num => $element) {
                $tags[$num] = PorterStemmer::Stem(strtolower(trim($element)));
                //stem elements
            }
            $tags = array_filter(array_unique($tags));
            //delete duplicates and empty elements
            //create the entry for the current document in the doc-term tags matrix
            foreach ($tags as $tag) {
                $tags_dt[$guid][$tag] = 1;
            }
        }
    }
    file_put_contents($IOdir . "tags_dt_raw", serialize($tags_dt));
    if (PHP_OS == "Linux") {
        chmod($IOdir . "tags_dt_raw", 0666);
    }
    //set rw permissions for everybody for this file
    if ($enable_synonyms) {
        foreach ($keys as $num => $key) {
            if (!isset($keys[$num])) {
                continue;
            }
            //since there is an unset on this array into the foreach, we have to check if the present key is still available or not
            foreach ($keys as $num2 => $key2) {
                if (!isset($keys[$num2])) {
                    continue;
                }
                //since there is an unset on this array into the foreach, we have to check if the present key is still available or not
                if ($key != $key2 && check_synonyms($key, $key2)) {
                    echo "\n{$key} and {$key2} are synonyms\n";
                    foreach ($tags_dt as $guid => $element) {
                        if (isset($tags_dt[$guid][$key2])) {
                            unset($tags_dt[$guid][$key2]);
                            $tags_dt[$guid][$key] = 1;
                        }
                    }
                    unset($keys[$num2]);
                }
            }
        }
    }
    //print_r($tags_dt);
    echo 'TAGS Doc-term matrix created' . "\n";
    return $tags_dt;
}