PHP phutil_is_utf8函数代码示例

/**
 * Convert a string into valid UTF-8. This function is quite slow.
 *
 * When invalid byte subsequences are encountered, they will be replaced with
 * U+FFFD, the Unicode replacement character.
 *
 * @param   string  String to convert to valid UTF-8.
 * @return  string  String with invalid UTF-8 byte subsequences replaced with
 *                  U+FFFD.
 */
function phutil_utf8ize($string)
{
    if (phutil_is_utf8($string)) {
        return $string;
    }
    // There is no function to do this in iconv, mbstring or ICU to do this, so
    // do it (very very slowly) in pure PHP.
    // TODO: Provide an optional fast C implementation ala fb_utf8ize() if this
    // ever shows up in profiles?
    // NOTE: Overlong 3-byte and 4-byte representations incorrectly survive
    // this function.
    $result = array();
    $regex = "/([-]" . "|[Â-ß][€-¿]" . "|[à-ï][€-¿][€-¿]" . "|[ð-ô][€-¿][€-¿][€-¿])" . "|(.)/";
    $offset = 0;
    $matches = null;
    while (preg_match($regex, $string, $matches, 0, $offset)) {
        if (!isset($matches[2])) {
            $result[] = $matches[1];
        } else {
            // Unicode replacement character, U+FFFD.
            $result[] = "�";
        }
        $offset += strlen($matches[0]);
    }
    return implode('', $result);
}

 /**
  * Check inserts for characters outside of the BMP. Even with the strictest
  * settings, MySQL will silently truncate data when it encounters these, which
  * can lead to data loss and security problems.
  */
 protected function validateUTF8String($string)
 {
     if (phutil_is_utf8($string)) {
         return;
     }
     throw new AphrontCharacterSetQueryException(pht('Attempting to construct a query using a non-utf8 string when ' . 'utf8 is expected. Use the `%%B` conversion to escape binary ' . 'strings data.'));
 }

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
if ($argc > 1) {
    $_SERVER['PHABRICATOR_ENV'] = $argv[1];
}
$root = dirname(dirname(dirname(__FILE__)));
require_once $root . '/scripts/__init_script__.php';
require_once $root . '/externals/mimemailparser/MimeMailParser.class.php';
$parser = new MimeMailParser();
$parser->setText(file_get_contents('php://stdin'));
$text_body = $parser->getMessageBody('text');
$text_body_headers = $parser->getMessageBodyHeaders('text');
$content_type = idx($text_body_headers, 'content-type');
if (!phutil_is_utf8($text_body) && (preg_match('/charset="(.*?)"/', $content_type, $matches) || preg_match('/charset=(\\S+)/', $content_type, $matches))) {
    $text_body = phutil_utf8_convert($text_body, "UTF-8", $matches[1]);
}
$headers = $parser->getHeaders();
$headers['subject'] = iconv_mime_decode($headers['subject'], 0, "UTF-8");
$headers['from'] = iconv_mime_decode($headers['from'], 0, "UTF-8");
$received = new PhabricatorMetaMTAReceivedMail();
$received->setHeaders($headers);
$received->setBodies(array('text' => $text_body, 'html' => $parser->getMessageBody('html')));
$attachments = array();
foreach ($parser->getAttachments() as $attachment) {
    if (preg_match('@text/(plain|html)@', $attachment->getContentType()) && $attachment->getContentDisposition() == 'inline') {
        // If this is an "inline" attachment with some sort of text content-type,
        // do not treat it as a file for attachment. MimeMailParser already picked
        // it up in the getMessageBody() call above. We still want to treat 'inline'
        // attachments with other content types (e.g., images) as attachments.

//.........这里部分代码省略.........
         } else {
             if ($repository_api instanceof ArcanistMercurialAPI) {
                 $diff = $repository_api->getFullMercurialDiff();
                 if (!strlen($diff)) {
                     throw new ArcanistUsageException(pht('No changes found. (Did you specify the wrong commit range?)'));
                 }
                 $changes = $parser->parseDiff($diff);
             } else {
                 throw new Exception(pht('Repository API is not supported.'));
             }
         }
     }
     if (count($changes) > 250) {
         $message = pht('This diff has a very large number of changes (%s). Differential ' . 'works best for changes which will receive detailed human review, ' . 'and not as well for large automated changes or bulk checkins. ' . 'See %s for information about reviewing big checkins. Continue anyway?', phutil_count($changes), 'https://secure.phabricator.com/book/phabricator/article/' . 'differential_large_changes/');
         if (!phutil_console_confirm($message)) {
             throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
         }
     }
     $limit = 1024 * 1024 * 4;
     foreach ($changes as $change) {
         $size = 0;
         foreach ($change->getHunks() as $hunk) {
             $size += strlen($hunk->getCorpus());
         }
         if ($size > $limit) {
             $byte_warning = pht("Diff for '%s' with context is %s bytes in length. " . "Generally, source changes should not be this large.", $change->getCurrentPath(), new PhutilNumber($size));
             if (!$this->getArgument('less-context')) {
                 $byte_warning .= ' ' . pht("If this file is a huge text file, try using the '%s' flag.", '--less-context');
             }
             if ($repository_api instanceof ArcanistSubversionAPI) {
                 throw new ArcanistUsageException($byte_warning . ' ' . pht("If the file is not a text file, mark it as binary with:" . "\n\n  \$ %s\n", 'svn propset svn:mime-type application/octet-stream <filename>'));
             } else {
                 $confirm = $byte_warning . ' ' . pht("If the file is not a text file, you can mark it 'binary'. " . "Mark this file as 'binary' and continue?");
                 if (phutil_console_confirm($confirm)) {
                     $change->convertToBinaryChange($repository_api);
                 } else {
                     throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
                 }
             }
         }
     }
     $try_encoding = nonempty($this->getArgument('encoding'), null);
     $utf8_problems = array();
     foreach ($changes as $change) {
         foreach ($change->getHunks() as $hunk) {
             $corpus = $hunk->getCorpus();
             if (!phutil_is_utf8($corpus)) {
                 // If this corpus is heuristically binary, don't try to convert it.
                 // mb_check_encoding() and mb_convert_encoding() are both very very
                 // liberal about what they're willing to process.
                 $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
                 if (!$is_binary) {
                     if (!$try_encoding) {
                         try {
                             $try_encoding = $this->getRepositoryEncoding();
                         } catch (ConduitClientException $e) {
                             if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') {
                                 echo phutil_console_wrap(pht('Lookup of encoding in arcanist project failed: %s', $e->getMessage()) . "\n");
                             } else {
                                 throw $e;
                             }
                         }
                     }
                     if ($try_encoding) {
                         $corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
                         $name = $change->getCurrentPath();
                         if (phutil_is_utf8($corpus)) {
                             $this->writeStatusMessage(pht("Converted a '%s' hunk from '%s' to UTF-8.\n", $name, $try_encoding));
                             $hunk->setCorpus($corpus);
                             continue;
                         }
                     }
                 }
                 $utf8_problems[] = $change;
                 break;
             }
         }
     }
     // If there are non-binary files which aren't valid UTF-8, warn the user
     // and treat them as binary changes. See D327 for discussion of why Arcanist
     // has this behavior.
     if ($utf8_problems) {
         $utf8_warning = sprintf("%s\n\n%s\n\n    %s\n", pht('This diff includes %s file(s) which are not valid UTF-8 (they ' . 'contain invalid byte sequences). You can either stop this ' . 'workflow and fix these files, or continue. If you continue, ' . 'these files will be marked as binary.', phutil_count($utf8_problems)), pht("You can learn more about how Phabricator handles character " . "encodings (and how to configure encoding settings and detect and " . "correct encoding problems) by reading 'User Guide: UTF-8 and " . "Character Encoding' in the Phabricator documentation."), pht('%s AFFECTED FILE(S)', phutil_count($utf8_problems)));
         $confirm = pht('Do you want to mark these %s file(s) as binary and continue?', phutil_count($utf8_problems));
         echo phutil_console_format("**%s**\n", pht('Invalid Content Encoding (Non-UTF8)'));
         echo phutil_console_wrap($utf8_warning);
         $file_list = mpull($utf8_problems, 'getCurrentPath');
         $file_list = '    ' . implode("\n    ", $file_list);
         echo $file_list;
         if (!phutil_console_confirm($confirm, $default_no = false)) {
             throw new ArcanistUsageException(pht('Aborted workflow to fix UTF-8.'));
         } else {
             foreach ($utf8_problems as $change) {
                 $change->convertToBinaryChange($repository_api);
             }
         }
     }
     $this->uploadFilesForChanges($changes);
     return $changes;
 }

/**
 * Produce a human-readable explanation why a value can not be JSON-encoded.
 *
 * @param wild Value to validate.
 * @param string Path within the object to provide context.
 * @return string|null Explanation of why it can't be encoded, or null.
 */
function phutil_validate_json($value, $path = '')
{
    if ($value === null) {
        return;
    }
    if ($value === true) {
        return;
    }
    if ($value === false) {
        return;
    }
    if (is_int($value)) {
        return;
    }
    if (is_float($value)) {
        return;
    }
    if (is_array($value)) {
        foreach ($value as $key => $subvalue) {
            if (strlen($path)) {
                $full_key = $path . ' > ';
            } else {
                $full_key = '';
            }
            if (!phutil_is_utf8($key)) {
                $full_key = $full_key . phutil_utf8ize($key);
                return pht('Dictionary key "%s" is not valid UTF8, and can not be JSON encoded.', $full_key);
            }
            $full_key .= $key;
            $result = phutil_validate_json($subvalue, $full_key);
            if ($result !== null) {
                return $result;
            }
        }
    }
    if (is_string($value)) {
        if (!phutil_is_utf8($value)) {
            $display = substr($value, 0, 256);
            $display = phutil_utf8ize($display);
            if (!strlen($path)) {
                return pht('String value is not valid UTF8, and can not be JSON encoded: %s', $display);
            } else {
                return pht('Dictionary value at key "%s" is not valid UTF8, and can not be ' . 'JSON encoded: %s', $path, $display);
            }
        }
    }
    return;
}

//.........这里部分代码省略.........
             throw new ArcanistUsageException("Aborted generation of gigantic diff.");
         }
     }
     $limit = 1024 * 1024 * 4;
     foreach ($changes as $change) {
         $size = 0;
         foreach ($change->getHunks() as $hunk) {
             $size += strlen($hunk->getCorpus());
         }
         if ($size > $limit) {
             $file_name = $change->getCurrentPath();
             $change_size = number_format($size);
             $byte_warning = "Diff for '{$file_name}' with context is {$change_size} bytes in " . "length. Generally, source changes should not be this large.";
             if (!$this->getArgument('less-context')) {
                 $byte_warning .= " If this file is a huge text file, try using the " . "'--less-context' flag.";
             }
             if ($repository_api instanceof ArcanistSubversionAPI) {
                 throw new ArcanistUsageException("{$byte_warning} If the file is not a text file, mark it as " . "binary with:" . "\n\n" . "  \$ svn propset svn:mime-type application/octet-stream <filename>" . "\n");
             } else {
                 $confirm = "{$byte_warning} If the file is not a text file, you can " . "mark it 'binary'. Mark this file as 'binary' and continue?";
                 if (phutil_console_confirm($confirm)) {
                     $change->convertToBinaryChange();
                 } else {
                     throw new ArcanistUsageException("Aborted generation of gigantic diff.");
                 }
             }
         }
     }
     $try_encoding = nonempty($this->getArgument('encoding'), null);
     $utf8_problems = array();
     foreach ($changes as $change) {
         foreach ($change->getHunks() as $hunk) {
             $corpus = $hunk->getCorpus();
             if (!phutil_is_utf8($corpus)) {
                 // If this corpus is heuristically binary, don't try to convert it.
                 // mb_check_encoding() and mb_convert_encoding() are both very very
                 // liberal about what they're willing to process.
                 $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
                 if (!$is_binary) {
                     if (!$try_encoding) {
                         try {
                             $try_encoding = $this->getRepositoryEncoding();
                         } catch (ConduitClientException $e) {
                             if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') {
                                 echo phutil_console_wrap("Lookup of encoding in arcanist project failed\n" . $e->getMessage());
                             } else {
                                 throw $e;
                             }
                         }
                     }
                     if ($try_encoding && $try_encoding != 'UTF-8') {
                         if (!function_exists('mb_convert_encoding')) {
                             throw new ArcanistUsageException("This diff includes a file encoded in '{$try_encoding}', " . "but you don't have the PHP mbstring extension installed " . "so it can't be converted to UTF-8. Install mbstring.");
                         }
                         $corpus = mb_convert_encoding($corpus, 'UTF-8', $try_encoding);
                         $name = $change->getCurrentPath();
                         if (phutil_is_utf8($corpus)) {
                             $this->writeStatusMessage("Converted a '{$name}' hunk from '{$try_encoding}' " . "to UTF-8.\n");
                             $hunk->setCorpus($corpus);
                             continue;
                         }
                     }
                 }
                 $utf8_problems[] = $change;
                 break;
             }

 protected function detectEncodingForStorage($string)
 {
     return phutil_is_utf8($string) ? 'utf8' : null;
 }

//.........这里部分代码省略.........
         }
         $hunk->setOldLength($old_len);
         $hunk->setNewLength($new_len);
         $add = 0;
         $del = 0;
         $hit_next_hunk = false;
         while (($line = $this->nextLine()) !== null) {
             if (strlen(rtrim($line, "\r\n"))) {
                 $char = $line[0];
             } else {
                 // Normally, we do not encouter empty lines in diffs, because
                 // unchanged lines have an initial space. However, in Git, with
                 // the option `diff.suppress-blank-empty` set, unchanged blank lines
                 // emit as completely empty. If we encounter a completely empty line,
                 // treat it as a ' ' (i.e., unchanged empty line) line.
                 $char = ' ';
             }
             switch ($char) {
                 case '\\':
                     if (!preg_match('@\\ No newline at end of file@', $line)) {
                         $this->didFailParse(pht("Expected '\\ No newline at end of file'."));
                     }
                     if ($new_len) {
                         $real[] = $line;
                         $hunk->setIsMissingOldNewline(true);
                     } else {
                         $real[] = $line;
                         $hunk->setIsMissingNewNewline(true);
                     }
                     if (!$new_len) {
                         break 2;
                     }
                     break;
                 case '+':
                     ++$add;
                     --$new_len;
                     $real[] = $line;
                     break;
                 case '-':
                     if (!$old_len) {
                         // In this case, we've hit "---" from a new file. So don't
                         // advance the line cursor.
                         $hit_next_hunk = true;
                         break 2;
                     }
                     ++$del;
                     --$old_len;
                     $real[] = $line;
                     break;
                 case ' ':
                     if (!$old_len && !$new_len) {
                         break 2;
                     }
                     --$old_len;
                     --$new_len;
                     $real[] = $line;
                     break;
                 default:
                     // We hit something, likely another hunk.
                     $hit_next_hunk = true;
                     break 2;
             }
         }
         if ($old_len || $new_len) {
             $this->didFailParse(pht('Found the wrong number of hunk lines.'));
         }
         $corpus = implode('', $real);
         $is_binary = false;
         if ($this->detectBinaryFiles) {
             $is_binary = !phutil_is_utf8($corpus);
             $try_encoding = $this->tryEncoding;
             if ($is_binary && $try_encoding) {
                 $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
                 if (!$is_binary) {
                     $corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
                     if (!phutil_is_utf8($corpus)) {
                         throw new Exception(pht("Failed to convert a hunk from '%s' to UTF-8. " . "Check that the specified encoding is correct.", $try_encoding));
                     }
                 }
             }
         }
         if ($is_binary) {
             // SVN happily treats binary files which aren't marked with the right
             // mime type as text files. Detect that junk here and mark the file
             // binary. We'll catch stuff with unicode too, but that's verboten
             // anyway. If there are too many false positives with this we might
             // need to make it threshold-triggered instead of triggering on any
             // unprintable byte.
             $change->setFileType(ArcanistDiffChangeType::FILE_BINARY);
         } else {
             $hunk->setCorpus($corpus);
             $hunk->setAddLines($add);
             $hunk->setDelLines($del);
             $change->addHunk($hunk);
         }
         if (!$hit_next_hunk) {
             $line = $this->nextNonemptyLine();
         }
     } while (preg_match('/^@@ /', $line));
 }

 public function testUTF8BMP()
 {
     $tests = array("" => array(true, true, "empty string"), "a" => array(true, true, "a"), "a͠͠" => array(true, true, "a with combining"), "☃" => array(true, true, "snowman"), "" => array(true, true, "U+FFFF"), "��" => array(false, false, "Invalid, byte range."), "𐀀" => array(true, false, "U+10000"), "𝄞" => array(true, false, "gclef"), "musical 𝄞 g-clef" => array(true, false, "gclef text"), "�" => array(false, false, "Invalid, truncated."), "���" => array(false, false, "Nonminimal 3-byte character."), "�" => array(false, false, "Partial 2-byte character."), "�" => array(false, false, "Partial BMP 0xE0 character."), "�" => array(false, false, "Partial BMP cahracter."));
     foreach ($tests as $input => $test) {
         list($expect_utf8, $expect_bmp, $test_name) = $test;
         $this->assertEqual($expect_utf8, phutil_is_utf8($input), pht('is_utf(%s)', $test_name));
         $this->assertEqual($expect_bmp, phutil_is_utf8_with_only_bmp_characters($input), pht('is_utf_bmp(%s)', $test_name));
     }
 }

 private function buildCorpus($selected, $file_query, $needs_blame, $drequest, $path, $data)
 {
     $image_type = $this->getImageType($path);
     if ($image_type && !$selected) {
         $corpus = phutil_render_tag('img', array('style' => 'padding-bottom: 10px', 'src' => 'data:' . $image_type . ';base64,' . base64_encode($data)));
         return $corpus;
     }
     $document_type = $this->getDocumentType($path);
     if ($document_type && !$selected || !phutil_is_utf8($data)) {
         $data = $file_query->getRawData();
         $document_type_description = $document_type ? $document_type : 'binary';
         $corpus = phutil_render_tag('p', array('style' => 'text-align: center;'), phutil_render_tag('a', array('href' => '?view=raw', 'class' => 'button'), "View {$document_type_description}"));
         return $corpus;
     }
     // TODO: blame of blame.
     switch ($selected) {
         case 'plain':
             $style = "margin: 1em 2em; width: 90%; height: 80em; font-family: monospace";
             $corpus = phutil_render_tag('textarea', array('style' => $style), phutil_escape_html($file_query->getRawData()));
             break;
         case 'plainblame':
             $style = "margin: 1em 2em; width: 90%; height: 80em; font-family: monospace";
             list($text_list, $rev_list, $blame_dict) = $file_query->getBlameData();
             $rows = array();
             foreach ($text_list as $k => $line) {
                 $rev = $rev_list[$k];
                 if (isset($blame_dict[$rev]['handle'])) {
                     $author = $blame_dict[$rev]['handle']->getName();
                 } else {
                     $author = $blame_dict[$rev]['author'];
                 }
                 $rows[] = sprintf("%-10s %-20s %s", substr($rev, 0, 7), $author, $line);
             }
             $corpus = phutil_render_tag('textarea', array('style' => $style), phutil_escape_html(implode("\n", $rows)));
             break;
         case 'highlighted':
         case 'blame':
         default:
             require_celerity_resource('syntax-highlighting-css');
             list($text_list, $rev_list, $blame_dict) = $file_query->getBlameData();
             $text_list = implode("\n", $text_list);
             $text_list = PhabricatorSyntaxHighlighter::highlightWithFilename($path, $text_list);
             $text_list = explode("\n", $text_list);
             $rows = $this->buildDisplayRows($text_list, $rev_list, $blame_dict, $needs_blame, $drequest, $file_query, $selected);
             $corpus_table = phutil_render_tag('table', array('class' => "diffusion-source remarkup-code PhabricatorMonospaced"), implode("\n", $rows));
             $corpus = phutil_render_tag('div', array('style' => 'padding: 0pt 2em;'), $corpus_table);
             break;
     }
     return $corpus;
 }

function show(array $files, $context)
{
    foreach ($files as $file) {
        $data = read($file);
        $ok = phutil_is_utf8($data);
        if ($ok) {
            echo "OKAY";
        } else {
            echo "FAIL";
        }
        echo "  " . name($file) . "\n";
        if (!$ok) {
            $lines = explode("\n", $data);
            $len = count($lines);
            $map = array();
            $bad = array();
            foreach ($lines as $n => $line) {
                if (phutil_is_utf8($line)) {
                    continue;
                }
                $bad[$n] = true;
                for ($jj = max(0, $n - $context); $jj < min($len, $n + 1 + $context); $jj++) {
                    $map[$jj] = true;
                }
            }
            $width = strlen(max(array_keys($map)));
            // Set $last such that we print a newline on the first iteration thorugh
            // the loop.
            $last = -2;
            foreach ($map as $idx => $ignored) {
                if ($idx != $last + 1) {
                    printf("\n");
                }
                $last = $idx;
                $line = $lines[$idx];
                if (!empty($bad[$idx])) {
                    $line = show_problems($line);
                }
                printf("  % {$width}d  %s\n", $idx + 1, $line);
            }
            echo "\n";
        }
    }
    return 0;
}

//.........这里部分代码省略.........
         if (!strlen($new_len)) {
             $new_len = 1;
         }
         $hunk->setOldLength($old_len);
         $hunk->setNewLength($new_len);
         $add = 0;
         $del = 0;
         $advance = false;
         while (($line = $this->nextLine()) !== null) {
             if (strlen($line)) {
                 $char = $line[0];
             } else {
                 $char = '~';
             }
             switch ($char) {
                 case '\\':
                     if (!preg_match('@\\ No newline at end of file@', $line)) {
                         $this->didFailParse("Expected '\\ No newline at end of file'.");
                     }
                     if ($new_len) {
                         $real[] = $line;
                         $hunk->setIsMissingOldNewline(true);
                     } else {
                         $real[] = $line;
                         $hunk->setIsMissingNewNewline(true);
                     }
                     if (!$new_len) {
                         $advance = true;
                         break 2;
                     }
                     break;
                 case '+':
                     if (!$new_len) {
                         break 2;
                     }
                     ++$add;
                     --$new_len;
                     $real[] = $line;
                     break;
                 case '-':
                     if (!$old_len) {
                         break 2;
                     }
                     ++$del;
                     --$old_len;
                     $real[] = $line;
                     break;
                 case ' ':
                     if (!$old_len && !$new_len) {
                         break 2;
                     }
                     --$old_len;
                     --$new_len;
                     $real[] = $line;
                     break;
                 case '~':
                     $advance = true;
                     break 2;
                 default:
                     break 2;
             }
         }
         if ($old_len != 0 || $new_len != 0) {
             $this->didFailParse("Found the wrong number of hunk lines.");
         }
         $corpus = implode("\n", $real);
         $is_binary = false;
         if ($this->detectBinaryFiles) {
             $is_binary = !phutil_is_utf8($corpus);
             if ($is_binary && $this->tryEncoding) {
                 $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
                 if (!$is_binary) {
                     // NOTE: This feature is HIGHLY EXPERIMENTAL and will cause a lot
                     // of issues. Use it at your own risk.
                     $corpus = mb_convert_encoding($corpus, 'UTF-8', $this->tryEncoding);
                     if (!phutil_is_utf8($corpus)) {
                         throw new Exception('Failed converting hunk to ' . $this->tryEncoding);
                     }
                 }
             }
         }
         if ($is_binary) {
             // SVN happily treats binary files which aren't marked with the right
             // mime type as text files. Detect that junk here and mark the file
             // binary. We'll catch stuff with unicode too, but that's verboten
             // anyway. If there are too many false positives with this we might
             // need to make it threshold-triggered instead of triggering on any
             // unprintable byte.
             $change->setFileType(ArcanistDiffChangeType::FILE_BINARY);
         } else {
             $hunk->setCorpus($corpus);
             $hunk->setAddLines($add);
             $hunk->setDelLines($del);
             $change->addHunk($hunk);
         }
         if ($advance) {
             $line = $this->nextNonemptyLine();
         }
     } while (preg_match('/^@@ /', $line));
 }

 public function testUTF8BMP()
 {
     $tests = array('' => array(true, true, pht('empty string')), 'a' => array(true, true, 'a'), "a͠͠" => array(true, true, pht('%s with combining', 'a')), "☃" => array(true, true, pht('snowman')), "" => array(true, true, 'U+FFFF'), "��" => array(false, false, pht('Invalid, byte range.')), "����" => array(false, false, pht('Nonminimal 4-byte character.')), "𐀀" => array(true, false, 'U+10000'), "𝄞" => array(true, false, 'gclef'), "musical 𝄞 g-clef" => array(true, false, pht('gclef text')), "�" => array(false, false, pht('Invalid, truncated.')), "���" => array(false, false, pht('Nonminimal 3-byte character.')), "�" => array(false, false, pht('Partial 2-byte character.')), "�" => array(false, false, pht('Partial BMP 0xE0 character.')), "�" => array(false, false, pht('Partial BMP cahracter.')));
     foreach ($tests as $input => $test) {
         list($expect_utf8, $expect_bmp, $test_name) = $test;
         // Depending on what's installed on the system, this may use an
         // extension.
         $this->assertEqual($expect_utf8, phutil_is_utf8($input), pht('is_utf(%s)', $test_name));
         // Also test this against the pure PHP implementation, explicitly.
         $this->assertEqual($expect_utf8, phutil_is_utf8_slowly($input), pht('is_utf_slowly(%s)', $test_name));
         $this->assertEqual($expect_bmp, phutil_is_utf8_with_only_bmp_characters($input), pht('is_utf_bmp(%s)', $test_name));
     }
 }