PHP normalizer_normalize函数代码示例

 private static function cleanChars($book)
 {
     foreach ($book as $key => $value) {
         $value = normalizer_normalize($value, \Normalizer::FORM_C);
         $value = preg_replace('/\\x{009c}/u', '', $value);
         $value = preg_replace('/\\x{0098}/u', '', $value);
         //debug (json_encode($value));
         $book[$key] = $value;
     }
     return $book;
 }

 public static function clean($string)
 {
     $string = filter_var($string, FILTER_SANITIZE_STRING, FILTER_FLAG_NO_ENCODE_QUOTES);
     // Ensure bytestream is valid
     if (!mb_check_encoding($string, 'UTF-8')) {
         throw new \InvalidArgumentException('Invalid unicode input.');
     }
     // Clean and normalise unicode
     $string = iconv('UTF-8', 'UTF-8//IGNORE', $string);
     $string = normalizer_normalize($string);
     // Strip control characters
     $string = preg_replace('~\\p{C}+~u', '', $string);
     return $string;
 }

 /**
  * Normalizes the encoding of a string (UTF8 NFD to NFC).
  *
  * On HFS+ filesystem (OS X) filenames are stored in UTF8 NFD while all other filesystems are
  * using UTF8 NFC. NFC is more common in general.
  *
  * @param string $string Input string
  *
  * @return string
  */
 public static function normalizeEncoding($string)
 {
     static $normalizer;
     if (null === $normalizer) {
         if (function_exists('normalizer_normalize')) {
             $normalizer = function ($string) {
                 return normalizer_normalize($string, Normalizer::FORM_C);
             };
         } else {
             $normalizer = function ($string) {
                 return str_replace(["Ä", "ä", "Ö", "ö", "Ü", "ü"], ['Ä', 'ä', 'Ö', 'ö', 'Ü', 'ü'], $string);
             };
         }
     }
     return $normalizer($string);
 }

function strip_diacriticals($in)
{
    if (function_exists('transliterator_transliterate')) {
        // PHP 5.4 + intl
        return transliterator_transliterate('Any-Latin; Latin-ASCII', $in);
    }
    if (function_exists('normalizer_normalize')) {
        // PHP 5.3 + intl
        return normalizer_normalize(preg_replace('/\\p{Mn}+/u', '', normalizer_normalize($in, Normalizer::FORM_D)), Normalizer::FORM_C);
    }
    if (function_exists('iconv') && ICONV_IMPL == 'glibc') {
        return iconv('UTF-8', 'ASCII//TRANSLIT', $in);
    }
    // version incomplète (limitée au latin1)
    $patterns = array('~&([A-za-z])(?:grave|acute|circ|tilde|uml|ring|cedil|slash|caron);~' => '\\1', '~&([A-za-z]{2})lig;~' => '\\1', '~&[^;]+;~' => '');
    $out = htmlentities($in, ENT_NOQUOTES, 'UTF-8');
    $out = preg_replace(array_keys($patterns), array_values($patterns), $out);
    return $out;
}

 /**
  * Enhanced 'remove_accents'. If the php Normalizer extension installed, use it.
  *
  * @since 1.0.0
  *
  * @see remove_accents()
  *
  * @param string $string Text that might have accent characters
  * @return string Filtered string with replaced "nice" characters.
  */
 public function remove_accents($string)
 {
     if (function_exists('normalizer_normalize')) {
         if (!normalizer_is_normalized($string, Normalizer::FORM_C)) {
             $string = normalizer_normalize($string, Normalizer::FORM_C);
         }
     }
     return remove_accents($string);
 }

 public static function encodeRelativeDescriptorString($str)
 {
     if (function_exists('normalizer_normalize')) {
         $str = normalizer_normalize($str);
     }
     $str = Z_Unicode::convertCharStr2UTF8($str);
     // convertNumbers2Char($str, 'hex')
     $str = preg_replace_callback("/([A-Fa-f0-9]{2})/", function ($matches) {
         return Z_Unicode::hex2char($matches[0]);
     }, str_replace(" ", "", $str));
     return $str;
 }

 /**
  * Remove all unwanted caracters
  *
  * @param string $text
  *
  * @return string
  */
 public function normalize($text)
 {
     return preg_replace('/\\pM*/u', '', normalizer_normalize($text, \Normalizer::FORM_D));
 }

 /**
  * Tests that "normalizer_normalize" exists and works
  *
  * @return bool
  */
 public static function hasNormalizerSupport()
 {
     static $ret = null;
     if (null === $ret) {
         $form_c = "Å";
         // 'LATIN CAPITAL LETTER A WITH RING ABOVE' (U+00C5)
         $form_d = "Å";
         // A followed by 'COMBINING RING ABOVE' (U+030A)
         $ret = function_exists('normalizer_normalize') && $form_c === normalizer_normalize($form_d);
     }
     return $ret;
 }

 function searchable($text, $lang = false)
 {
     global $cfg;
     if (function_exists('normalizer_normalize')) {
         // Normalize text input :: remove diacritics and such
         $text = normalizer_normalize($text, Normalizer::FORM_C);
     } else {
         // As a lightweight compatiblity, use a lightweight C
         // normalizer with diacritic removal, thanks
         // http://ahinea.com/en/tech/accented-translate.html
         $tr = array("ä" => "a", "ñ" => "n", "ö" => "o", "ü" => "u", "ÿ" => "y");
         $text = strtr($text, $tr);
     }
     // Decompose compatible versions of characters (ä => ae)
     $tr = array("ß" => "ss", "Æ" => "AE", "æ" => "ae", "IJ" => "IJ", "ij" => "ij", "Œ" => "OE", "œ" => "oe", "Ð" => "D", "Đ" => "D", "ð" => "d", "đ" => "d", "Ħ" => "H", "ħ" => "h", "ı" => "i", "ĸ" => "k", "Ŀ" => "L", "Ł" => "L", "ŀ" => "l", "ł" => "l", "Ŋ" => "N", "ʼn" => "n", "ŋ" => "n", "Ø" => "O", "ø" => "o", "ſ" => "s", "Þ" => "T", "Ŧ" => "T", "þ" => "t", "ŧ" => "t", "ä" => "ae", "ö" => "oe", "ü" => "ue", "Ä" => "AE", "Ö" => "OE", "Ü" => "UE");
     $text = strtr($text, $tr);
     // Drop separated diacritics
     $text = preg_replace('/\\p{M}/u', '', $text);
     // Drop extraneous whitespace
     $text = preg_replace('/(\\s)\\s+/u', '$1', $text);
     // Drop leading and trailing whitespace
     $text = trim($text);
     if (false && class_exists('IntlBreakIterator')) {
         // Split by word boundaries
         if ($tokenizer = IntlBreakIterator::createWordInstance($lang ?: ($cfg ? $cfg->getSystemLanguage() : 'en_US'))) {
             $tokenizer->setText($text);
             $tokens = array();
             foreach ($tokenizer as $token) {
                 $tokens[] = $token;
             }
             $text = implode(' ', $tokens);
         }
     } else {
         // Approximate word boundaries from Unicode chart at
         // http://www.unicode.org/reports/tr29/#Word_Boundaries
         // Punt for now
     }
     return $text;
 }

    /**
     * Normalise a UTf-8 string to FORM_C, avoiding the pitfalls in PHP's
     * normalizer_normalize function.
     * @param string $string the input string.
     * @return string the normalised string.
     */
    protected static function safe_normalize($string) {
        if ($string === '') {
            return '';
        }

        if (!function_exists('normalizer_normalize')) {
            return $string;
        }

        $normalised = normalizer_normalize($string, Normalizer::FORM_C);
        if (is_null($normalised)) {
            // An error occurred in normalizer_normalize, but we have no idea what.
            debugging('Failed to normalise string: ' . $string, DEBUG_DEVELOPER);
            return $string; // Return the original string, since it is the best we have.
        }

        return $normalised;
    }

 /**
  *
  * @see \X501\StringPrep\StringPrep::prepare()
  * @param string $string UTF-8 encoded string
  * @return string
  */
 public function apply($string)
 {
     return normalizer_normalize($string, \Normalizer::NFKC);
 }

 protected static function normalizeValue($value) {
     return isset($value) ? preg_replace('/(\p{M})/ui', '', normalizer_normalize($value, Normalizer::FORM_D)) : NULL;
 }

function fewer_specials($w, $strip = "̄")
{
    $w = normalizer_normalize($w, Normalizer::FORM_D);
    $w = str_replace("æ", "ae", $w);
    $w = str_replace("œ", "oe", $w);
    $w = str_replace("Æ", "ae", $w);
    $w = str_replace("Œ", "oe", $w);
    $w = str_replace($strip, "", $w);
    return $w;
}

 /**
  * Normalize the given UTF-8 string
  *
  * @see http://stackoverflow.com/a/7934397/99923
  * @param string $string to normalize
  * @param int $form to normalize as
  * @return string
  */
 public static function normalize($string, $form = Normalizer::FORM_KD)
 {
     return normalizer_normalize($string, $form);
 }

function normalize_keyword($keyword)
{
    global $normalize_keywords, $keywords_remove_diacritics;
    //Normalize the text if function available
    if ($normalize_keywords && function_exists('normalizer_normalize')) {
        $keyword = normalizer_normalize($keyword);
    }
    if ($keywords_remove_diacritics) {
        $keyword = remove_accents($keyword);
    }
    return $keyword;
}