PHP UtfNormal类代码示例

 function processPage($row)
 {
     global $wgContLang;
     $source = $row->img_name;
     if ($source == '') {
         // Ye olde empty rows. Just kill them.
         $this->killRow($source);
         return $this->progress(1);
     }
     $cleaned = $source;
     // About half of old bad image names have percent-codes
     $cleaned = rawurldecode($cleaned);
     // Some are old latin-1
     $cleaned = $wgContLang->checkTitleEncoding($cleaned);
     // Many of remainder look like non-normalized unicode
     $cleaned = UtfNormal::cleanUp($cleaned);
     $title = Title::makeTitleSafe(NS_IMAGE, $cleaned);
     if (is_null($title)) {
         $this->log("page {$source} ({$cleaned}) is illegal.");
         $safe = $this->buildSafeTitle($cleaned);
         $this->pokeFile($source, $safe);
         return $this->progress(1);
     }
     if ($title->getDbKey() !== $source) {
         $munged = $title->getDbKey();
         $this->log("page {$source} ({$munged}) doesn't match self.");
         $this->pokeFile($source, $munged);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 /**
  * Format an XML element as with self::element(), but run text through the
  * UtfNormal::cleanUp() validator first to ensure that no invalid UTF-8
  * is passed.
  *
  * @param $element String:
  * @param $attribs Array: Name=>value pairs. Values will be escaped.
  * @param $contents String: NULL to make an open tag only; '' for a contentless closed tag (default)
  * @return string
  */
 public static function elementClean($element, $attribs = array(), $contents = '')
 {
     if ($attribs) {
         $attribs = array_map(array('UtfNormal', 'cleanUp'), $attribs);
     }
     if ($contents) {
         $contents = UtfNormal::cleanUp($contents);
     }
     return self::element($element, $attribs, $contents);
 }

 /**
  * Format an XML element as with self::element(), but run text through the
  * UtfNormal::cleanUp() validator first to ensure that no invalid UTF-8
  * is passed.
  *
  * @param $element String:
  * @param $attribs Array: Name=>value pairs. Values will be escaped.
  * @param $contents String: NULL to make an open tag only; '' for a contentless closed tag (default)
  * @return string
  */
 public static function elementClean($element, $attribs = array(), $contents = '')
 {
     if ($attribs) {
         $attribs = array_map(array('UtfNormal', 'cleanUp'), $attribs);
     }
     if ($contents) {
         wfProfileIn(__METHOD__ . '-norm');
         $contents = UtfNormal::cleanUp($contents);
         wfProfileOut(__METHOD__ . '-norm');
     }
     return self::element($element, $attribs, $contents);
 }

function gs_utf8_decompose_to_ascii($str)
{
    static $map = null;
    if (!is_array($map)) {
        $map = _gs_utf8_get_map();
    }
    $str = UtfNormal::toNFD(strTr($str, $map));
    # return "safe" ASCII without control chars, newlines etc.
    //$str = preg_replace('/[^a-z0-9\-_. *#\'"!$()\/]/i', '', $str);
    $str = preg_replace('/[^\\x20-\\x7E]/', '', $str);
    return $str;
}

 function processPage($row)
 {
     $current = Title::makeTitle($row->wl_namespace, $row->wl_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if ($row->wl_user == 0 || is_null($title) || !$title->equals($current)) {
         $this->log("invalid watch by {$row->wl_user} for ({$row->wl_namespace}, \"{$row->wl_title}\")");
         $this->removeWatch($row);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 public static function formatDiffRow($title, $oldid, $newid, $timestamp, $comment, $actiontext = '')
 {
     global $wgFeedDiffCutoff, $wgContLang, $wgUser;
     wfProfileIn(__FUNCTION__);
     $skin = $wgUser->getSkin();
     # log enties
     $completeText = '<p>' . implode(' ', array_filter(array($actiontext, $skin->formatComment($comment)))) . "</p>\n";
     //NOTE: Check permissions for anonymous users, not current user.
     //      No "privileged" version should end up in the cache.
     //      Most feed readers will not log in anway.
     $anon = new User();
     $accErrors = $title->getUserPermissionsErrors('read', $anon, true);
     if ($title->getNamespace() >= 0 && !$accErrors) {
         if ($oldid) {
             wfProfileIn(__FUNCTION__ . "-dodiff");
             #$diffText = $de->getDiff( wfMsg( 'revisionasof',
             #	$wgContLang->timeanddate( $timestamp ) ),
             #	wfMsg( 'currentrev' ) );
             // Don't bother generating the diff if we won't be able to show it
             if ($wgFeedDiffCutoff > 0) {
                 $de = new DifferenceEngine($title, $oldid, $newid);
                 $diffText = $de->getDiff(wfMsg('previousrevision'), wfMsg('revisionasof', $wgContLang->timeanddate($timestamp)));
             }
             if (strlen($diffText) > $wgFeedDiffCutoff || $wgFeedDiffCutoff <= 0) {
                 // Omit large diffs
                 $diffLink = $title->escapeFullUrl('diff=' . $newid . '&oldid=' . $oldid);
                 $diffText = '<a href="' . $diffLink . '">' . htmlspecialchars(wfMsgForContent('showdiff')) . '</a>';
             } elseif ($diffText === false) {
                 // Error in diff engine, probably a missing revision
                 $diffText = "<p>Can't load revision {$newid}</p>";
             } else {
                 // Diff output fine, clean up any illegal UTF-8
                 $diffText = UtfNormal::cleanUp($diffText);
                 $diffText = self::applyDiffStyle($diffText);
             }
             wfProfileOut(__FUNCTION__ . "-dodiff");
         } else {
             $rev = Revision::newFromId($newid);
             if (is_null($rev)) {
                 $newtext = '';
             } else {
                 $newtext = $rev->getText();
             }
             $diffText = '<p><b>' . wfMsg('newpage') . '</b></p>' . '<div>' . nl2br(htmlspecialchars($newtext)) . '</div>';
         }
         $completeText .= $diffText;
     }
     wfProfileOut(__FUNCTION__);
     return $completeText;
 }

 /**
  * Returns the normalized form of the given page title, using the normalization rules of the given site.
  * If the given title is a redirect, the redirect weill be resolved and the redirect target is returned.
  *
  * @note  : This actually makes an API request to the remote site, so beware that this function is slow and depends
  *          on an external service.
  *
  * @note  : If MW_PHPUNIT_TEST is defined, the call to the external site is skipped, and the title
  *          is normalized using the local normalization rules as implemented by the Title class.
  *
  * @see Site::normalizePageName
  *
  * @since 1.21
  *
  * @param string $pageName
  *
  * @return string
  * @throws MWException
  */
 public function normalizePageName($pageName)
 {
     // Check if we have strings as arguments.
     if (!is_string($pageName)) {
         throw new MWException('$pageName must be a string');
     }
     // Go on call the external site
     if (defined('MW_PHPUNIT_TEST')) {
         // If the code is under test, don't call out to other sites, just normalize locally.
         // Note: this may cause results to be inconsistent with the actual normalization used by the respective remote site!
         $t = Title::newFromText($pageName);
         return $t->getPrefixedText();
     } else {
         // Make sure the string is normalized into NFC (due to the bug 40017)
         // but do nothing to the whitespaces, that should work appropriately.
         // @see https://bugzilla.wikimedia.org/show_bug.cgi?id=40017
         $pageName = UtfNormal::cleanUp($pageName);
         // Build the args for the specific call
         $args = array('action' => 'query', 'prop' => 'info', 'redirects' => true, 'converttitles' => true, 'format' => 'json', 'titles' => $pageName);
         $url = $this->getFileUrl('api.php') . '?' . wfArrayToCgi($args);
         // Go on call the external site
         //@todo: we need a good way to specify a timeout here.
         $ret = Http::get($url);
     }
     if ($ret === false) {
         wfDebugLog("MediaWikiSite", "call to external site failed: {$url}");
         return false;
     }
     $data = FormatJson::decode($ret, true);
     if (!is_array($data)) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned bad json: " . $ret);
         return false;
     }
     $page = static::extractPageRecord($data, $pageName);
     if (isset($page['missing'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned a marker for a missing page title! " . $ret);
         return false;
     }
     if (isset($page['invalid'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned a marker for an invalid page title! " . $ret);
         return false;
     }
     if (!isset($page['title'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> did not return a page title! " . $ret);
         return false;
     }
     return $page['title'];
 }

 function processPage($row)
 {
     $current = Title::makeTitle($row->page_namespace, $row->page_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if (is_null($title)) {
         $this->log("page {$row->page_id} ({$display}) is illegal.");
         $this->moveIllegalPage($row);
         return $this->progress(1);
     }
     if (!$title->equals($current)) {
         $this->log("page {$row->page_id} ({$display}) doesn't match self.");
         $this->moveInconsistentPage($row, $title);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 function makeInputText($max = false)
 {
     if ($max === false) {
         $max = $this->maxLength;
     }
     $length = mt_rand($this->minLength, $max);
     $s = '';
     for ($i = 0; $i < $length; $i++) {
         $hairIndex = mt_rand(0, count($this->hairs) - 1);
         $s .= $this->hairs[$hairIndex];
     }
     // Send through the UTF-8 normaliser
     // This resolves a few differences between the old preprocessor and the
     // XML-based one, which doesn't like illegals and converts line endings.
     // It's done by the MW UI, so it's a reasonably legitimate thing to do.
     $s = UtfNormal::cleanUp($s);
     return $s;
 }

 function processPage($row)
 {
     $current = Title::makeTitle($row->page_namespace, $row->page_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if (!is_null($title) && $title->equals($current) && $title->canExist()) {
         return $this->progress(0);
         // all is fine
     }
     if ($row->page_namespace == NS_FILE && $this->fileExists($row->page_title)) {
         $this->log("file {$row->page_title} needs cleanup, please run cleanupImages.php.");
         return $this->progress(0);
     } elseif (is_null($title)) {
         $this->log("page {$row->page_id} ({$display}) is illegal.");
         $this->moveIllegalPage($row);
         return $this->progress(1);
     } else {
         $this->log("page {$row->page_id} ({$display}) doesn't match self.");
         $this->moveInconsistentPage($row, $title);
         return $this->progress(1);
     }
 }

 /**
  * Function executed by use of {{#infoboxbuilder:}} parser function.
  * It gets the code from InfoboxBuilder.lua and creates new module object
  * from it. The module is then invoked and the result is returned.
  * @param  Parser  $parser Parser object
  * @param  PPFrame $frame  PPFrame object
  * @param  array   $args   Array of arguments passed from $frame object
  * @return string          A string returned by InfoboxBuilder.lua
  */
 public static function parserFunctionHook(\Parser $parser, $frame, $args)
 {
     wfProfileIn(__METHOD__);
     try {
         /**
          * Add the registered SCSS with the default theme
          */
         $parser->getOutput()->addModuleStyles('ext.wikia.InfoboxBuilder');
         $engine = \Scribunto::getParserEngine($parser);
         unset($args[0]);
         $childFrame = $frame->newChild($args, $parser->getTitle(), 1);
         $moduleText = file_get_contents(__DIR__ . '/includes/lua/InfoboxBuilder.lua');
         $module = new \Scribunto_LuaModule($engine, $moduleText, 'InfoboxBuilder');
         $result = $module->invoke('builder', $childFrame);
         $result = \UtfNormal::cleanUp(strval($result));
         wfProfileOut(__METHOD__);
         return $result;
     } catch (\ScribuntoException $e) {
         $trace = $e->getScriptTraceHtml(array('msgOptions' => array('content')));
         $html = \Html::element('p', array(), $e->getMessage());
         if ($trace !== false) {
             $html .= \Html::element('p', array(), wfMessage('scribunto-common-backtrace')->inContentLanguage()->text()) . $trace;
         }
         $out = $parser->getOutput();
         if (!isset($out->scribunto_errors)) {
             $out->addOutputHook('ScribuntoError');
             $out->scribunto_errors = array();
             $parser->addTrackingCategory('scribunto-common-error-category');
         }
         $out->scribunto_errors[] = $html;
         $id = 'mw-scribunto-error-' . (count($out->scribunto_errors) - 1);
         $parserError = wfMessage('scribunto-parser-error')->inContentLanguage()->text() . $parser->insertStripItem('<!--' . htmlspecialchars($e->getMessage()) . '-->');
         wfProfileOut(__METHOD__);
         // #iferror-compatible error element
         return "<strong class=\"error\"><span class=\"scribunto-error\" id=\"{$id}\">" . $parserError . "</span></strong>";
     }
 }

 /**
  * Get Normalized metadata in PHP-serialized form
  *
  * @param stdClass $video
  * @return string
  */
 protected static function getNormalizedMetadata($video)
 {
     // image.img_metadata
     $metadata = unserialize($video->img_metadata);
     foreach (self::$metadataFieldsContainingName as $field) {
         if (isset($metadata[$field])) {
             $metadata[$field] = \UtfNormal::toNFC($metadata[$field]);
         }
     }
     return serialize($metadata);
 }

 /**
  * Convert a UTF-8 string to normal form C. In Malayalam and Arabic, this
  * also cleans up certain backwards-compatible sequences, converting them
  * to the modern Unicode equivalent.
  *
  * This is language-specific for performance reasons only.
  *
  * @param $s string
  *
  * @return string
  */
 function normalize($s)
 {
     global $wgAllUnicodeFixes;
     $s = UtfNormal::cleanUp($s);
     if ($wgAllUnicodeFixes) {
         $s = $this->transformUsingPairFile('normalize-ar.ser', $s);
         $s = $this->transformUsingPairFile('normalize-ml.ser', $s);
     }
     return $s;
 }

function testLine($test, $line, &$total, &$success, &$failed, $columns, $exceptions, $verbose)
{
    $stripped = $line;
    UtfNormal::quickisNFCVerify($stripped);
    $same = $line == $stripped;
    $len = mb_strlen(substr($stripped, 0, strpos($stripped, '|')));
    if ($len == 0) {
        $len = strlen(substr($stripped, 0, strpos($stripped, '|')));
    }
    $ok = $same ^ $test >= 3;
    $ok ^= in_array($test, $exceptions);
    $ok &= $columns == $len;
    $total++;
    if ($ok) {
        $success++;
    } else {
        $failed++;
    }
    if ($verbose || !$ok) {
        print str_replace("\n", "{$len}\n", $stripped);
    }
}

 /**
  * Callback function for cleanUpUTF8()
  */
 private static function cleanUp_helper(&$s)
 {
     if (!is_string($s)) {
         return;
     }
     $s = UtfNormal::cleanUp($s);
 }