PHP StringUtil::unicodeToLatin方法代码示例

 public static function dist($s1, $s2)
 {
     $s1 = mb_strtolower(StringUtil::unicodeToLatin($s1));
     $s2 = mb_strtolower(StringUtil::unicodeToLatin($s2));
     $len1 = mb_strlen($s1);
     $len2 = mb_strlen($s2);
     // Split the strings into characters to minimize the number calls to getCharAt().
     $chars1 = array();
     for ($i = 0; $i < $len1; $i++) {
         $chars1[] = StringUtil::getCharAt($s1, $i);
     }
     $chars2 = array();
     for ($j = 0; $j < $len2; $j++) {
         $chars2[] = StringUtil::getCharAt($s2, $j);
     }
     // Initialize the first row and column of the matrix
     $a = array();
     for ($i = 0; $i <= $len1; $i++) {
         $a[$i][0] = $i * self::$DIST_OTHER;
     }
     for ($j = 0; $j <= $len2; $j++) {
         $a[0][$j] = $j * self::$COST_DEL;
     }
     // Compute the rest of the matrix with the custom Levenshtein algorithm
     for ($i = 0; $i < $len1; $i++) {
         for ($j = 0; $j < $len2; $j++) {
             $mati = $i + 1;
             $matj = $j + 1;
             // Delete
             $a[$mati][$matj] = $a[$mati][$matj - 1] + self::$COST_DEL;
             // Insert
             $costInsert = $i == 0 ? self::$INFTY : max(self::$COST_INS, self::letterDistance($chars1[$i], $chars1[$i - 1]));
             // At least COST_INS
             $a[$mati][$matj] = min($a[$mati][$matj], $a[$mati - 1][$matj] + $costInsert);
             // Modify (This includes the case where $s1[i] == $s2[j] because dist(x, x) returns 0)
             $a[$mati][$matj] = min($a[$mati][$matj], $a[$mati - 1][$matj - 1] + self::letterDistance($chars1[$i], $chars2[$j]));
             // Transpose
             if ($i && $j && $chars1[$i] == $chars2[$j - 1] && $chars1[$i - 1] == $chars2[$j]) {
                 $a[$mati][$matj] = min($a[$mati][$matj], $a[$mati - 2][$matj - 2] + self::$COST_TRANSPOSE);
             }
         }
     }
     return $a[$len1][$len2];
 }

 static function isSeparator($ch)
 {
     crawlerLog("INSIDE " . __FILE__ . ' - ' . __CLASS__ . '::' . __FUNCTION__ . '() - ' . 'line ' . __LINE__);
     return !(ctype_alpha(StringUtil::unicodeToLatin($ch)) || $ch == '-');
 }

assertEquals(AdminStringUtil::shorthandToUnicode("'~a'~A~'a~'A"), 'ắẮắẮ');
assertEquals(AdminStringUtil::shorthandToUnicode("~a~A^a^A'a'A"), 'ăĂâÂáÁ');
assertEquals(AdminStringUtil::shorthandToUnicode("`a`A:a:A"), 'àÀäÄ');
assertEquals(AdminStringUtil::shorthandToUnicode(",c,C'c'C~c~C"), 'çÇćĆčČ');
assertEquals(AdminStringUtil::shorthandToUnicode("'e'E`e`E^e^E"), 'éÉèÈêÊ');
assertEquals(AdminStringUtil::shorthandToUnicode(":e:E~e~E~g~G"), 'ëËĕĔğĞ');
assertEquals(AdminStringUtil::shorthandToUnicode("'^i'^I^'i^'I"), 'î́Î́î́Î́');
assertEquals(AdminStringUtil::shorthandToUnicode("'i'I`i`I^i^I"), 'íÍìÌîÎ');
assertEquals(AdminStringUtil::shorthandToUnicode(":i:I~i~I~n~N"), 'ïÏĭĬñÑ');
assertEquals(AdminStringUtil::shorthandToUnicode("'o'O`o`O^o^O"), 'óÓòÒôÔ');
assertEquals(AdminStringUtil::shorthandToUnicode(":o:O~o~O~r~R"), 'öÖõÕřŘ');
assertEquals(AdminStringUtil::shorthandToUnicode("~s~S,s,S,t,T"), 'šŠșȘțȚ');
assertEquals(AdminStringUtil::shorthandToUnicode("'u'U`u`U^u^U"), 'úÚùÙûÛ');
assertEquals(AdminStringUtil::shorthandToUnicode(":u:U~u~U"), 'üÜŭŬ');
assertEquals(AdminStringUtil::shorthandToUnicode("'y'Y:y:Y~z~Z"), 'ýÝÿŸžŽ');
assertEquals('acegyzACEGYZ', StringUtil::unicodeToLatin("ắčèğýžẮČÈĞÝŽ"));
assertEquals('mama', mb_strtolower('mama'));
assertEquals('mama', mb_strtolower('maMa'));
assertEquals('mama', mb_strtolower('MAmA'));
assertEquals('mamă', mb_strtolower('MAmă'));
assertEquals('mamă', mb_strtolower('MAmĂ'));
assertEquals('abcúùû', mb_strtolower('ABCÚÙÛ'));
assertEquals('ÿ', mb_strtolower('Ÿ'));
assertEquals('MAMA', mb_strtoupper('MAMA'));
assertEquals('MAMA', mb_strtoupper('MAmA'));
assertEquals('MAMA', mb_strtoupper('MAmA'));
assertEquals('MAMĂ', mb_strtoupper('MamĂ'));
assertEquals('MAMĂ', mb_strtoupper('maMă'));
assertEquals('ABCÚÙÛ', mb_strtoupper('abcúùû'));
assertEquals('Ÿ', mb_strtoupper('ÿ'));
// Check that we're using the right encoding

 function postProcess($fileName)
 {
     $tmpFile = tempnam($this->tmpDir, 'loc_');
     log_scriptLog('* removing diacritics');
     $s = file_get_contents($fileName);
     $s = StringUtil::unicodeToLatin($s);
     file_put_contents($tmpFile, $s);
     log_scriptLog('* removing duplicates and sorting');
     OS::executeAndAssert("sort -u {$tmpFile} -o {$fileName}");
     unlink($tmpFile);
 }

function stripImageName($fileName)
{
    $s = StringUtil::unicodeToLatin($fileName);
    $s = str_replace(array('-', ' ', 'ş', 'ţ', 'Ş', 'Ţ'), array('', '', 's', 't', 's', 't'), $s);
    $s = mb_strtolower($s);
    return $s;
}

function fetchNextRow($row)
{
    global $lexemDbResult;
    global $sourceMap;
    global $currentLexem;
    $def = Model::factory('Definition')->create($row);
    $def->internalRep = AdminStringUtil::xmlizeRequired($def->internalRep);
    if (hasFlag('d')) {
        $def->internalRep = AdminStringUtil::xmlizeOptional($def->internalRep);
    }
    $lexemNames = array();
    $lexemLatinNames = array();
    while (merge_compare($def, $currentLexem) < 0) {
        $currentLexem = $lexemDbResult->fetch();
    }
    while (merge_compare($def, $currentLexem) == 0) {
        $lexemNames[] = $currentLexem[1];
        $lexemLatinNames[] = StringUtil::unicodeToLatin($currentLexem[1]);
        $currentLexem = $lexemDbResult->fetch();
    }
    SmartyWrap::assign('def', $def);
    SmartyWrap::assign('lexemNames', $lexemNames);
    SmartyWrap::assign('lexemLatinNames', $lexemLatinNames);
    SmartyWrap::assign('source', $sourceMap[$def->sourceId]);
    SmartyWrap::assign('user', userCache_get($def->userId));
}

<?php

require_once "../phplib/util.php";
ini_set('max_execution_time', '3600');
ini_set("memory_limit", "128000000");
$query = "select I.formNoAccent from InflectedForm I, Lexem L, Model M, ModelDescription MD, ModelType MT " . "where I.lexemId = L.id and L.modelType = MT.code and MT.canonical = M.modelType and L.modelNumber = M.number and M.id = MD.modelId " . "and MD.inflectionId = I.inflectionId and MD.variant = I.variant and MD.applOrder = 0 and L.isLoc and MD.isLoc " . "and char_length(I.formNoAccent) between 2 and 15";
$dbResult = db_execute($query);
$fileName = tempnam('/tmp', 'unique_');
$file = fopen($fileName, 'w');
foreach ($dbResult as $dbRow) {
    fwrite($file, "{$dbRow[0]}\r\n");
}
fclose($file);
$s = file_get_contents($fileName);
$s = StringUtil::unicodeToLatin($s);
$s = strtoupper($s);
$file = fopen($fileName, 'w');
fwrite($file, $s);
fclose($file);
$fileName2 = tempnam('/tmp', 'unique_');
OS::executeAndAssert("sort -u {$fileName} -o {$fileName2}");
header('Content-type: text/plain');
print file_get_contents($fileName2);
OS::executeAndAssert("rm -f {$fileName} {$fileName2}");