PHP token_get_all函数代码示例

 public function format($source)
 {
     $this->tkns = token_get_all($source);
     $this->code = '';
     $tknsLen = sizeof($this->tkns);
     $touchedDoubleColon = false;
     for ($ptr = 0; $ptr < $tknsLen; ++$ptr) {
         $token = $this->tkns[$ptr];
         list($id) = $this->getToken($token);
         if (T_DOUBLE_COLON == $id) {
             $touchedDoubleColon = true;
         }
         if ($touchedDoubleColon && T_CLASS == $id) {
             $touchedDoubleColon = false;
             break;
         }
         if (T_CLASS == $id || T_INTERFACE == $id || T_TRAIT == $id) {
             $this->refWalkUsefulUntil($this->tkns, $ptr, T_STRING);
             list(, $name) = $this->getToken($this->tkns[$ptr]);
             $this->refWalkUsefulUntil($this->tkns, $ptr, ST_CURLY_OPEN);
             $start = $ptr;
             $this->refWalkCurlyBlock($this->tkns, $ptr);
             $end = ++$ptr;
             $this->convertToPlaceholder($name, $start, $end);
             break;
         }
     }
     return $this->render();
 }

function find_methods($source_code, $file)
{
    $class_name = "";
    $tokens = token_get_all($source_code);
    $in_token = false;
    foreach ($tokens as $key => $token) {
        switch ($token[0]) {
            case T_FUNCTION:
                $in_token = T_FUNCTION;
                continue;
                break;
            case T_STRING:
                switch ($in_token) {
                    case T_FUNCTION:
                        $line_num = $token[2];
                        $value = $token[1];
                        $token_type = $token[0];
                        $class_name .= $line_num . " " . $value . " " . $dir . '/' . $ENV['TM_FILEPATH'] . "\n";
                        $in_token = false;
                        break;
                }
                break;
            default:
                if (count($token) > 2) {
                    //$class_name.= "type " . token_name ($token[0]) . " line: ". $token[2] . " class " . $token[1];
                }
                break;
        }
    }
    echo $class_name;
}

 protected function loadFile($file)
 {
     $striped = "";
     // remove all whitespaces and comments
     // replace short tags
     $t = token_get_all(str_replace('<?=', '<?php echo ', php_strip_whitespace($file)));
     $blacklist = array(T_AS, T_CASE, T_INSTANCEOF, T_USE);
     foreach ($t as $i => $token) {
         if (is_string($token)) {
             $striped .= $token;
         } elseif (T_WHITESPACE === $token[0]) {
             if (isset($t[$i + 1]) && is_array($t[$i + 1])) {
                 if (in_array($t[$i + 1][0], $blacklist)) {
                     $striped .= $t[$i][1];
                     continue;
                 }
                 if (isset($t[$i - 1]) && is_array($t[$i - 1])) {
                     if (in_array($t[$i - 1][0], $blacklist)) {
                         $striped .= $t[$i][1];
                         continue;
                     }
                     if ($t[$i - 1][0] == T_ECHO || $t[$i - 1][0] == T_PRINT) {
                         $striped .= $t[$i][1];
                         continue;
                     }
                 }
             }
             $striped .= str_replace(' ', '', $token[1]);
         } else {
             $striped .= $token[1];
         }
     }
     $this->buffer = $striped;
 }

 public function format($source)
 {
     $this->tkns = token_get_all($source);
     $this->code = '';
     while (list($index, $token) = each($this->tkns)) {
         list($id, $text) = $this->getToken($token);
         $this->ptr = $index;
         if (T_WHITESPACE == $id || T_VARIABLE == $id || T_INLINE_HTML == $id || T_COMMENT == $id || T_DOC_COMMENT == $id || T_CONSTANT_ENCAPSED_STRING == $id) {
             $this->appendCode($text);
             continue;
         }
         if (T_STRING == $id && $this->leftUsefulTokenIs([T_DOUBLE_COLON, T_OBJECT_OPERATOR])) {
             $this->appendCode($text);
             continue;
         }
         if (T_START_HEREDOC == $id) {
             $this->appendCode($text);
             $this->printUntil(ST_SEMI_COLON);
             continue;
         }
         if (ST_QUOTE == $id) {
             $this->appendCode($text);
             $this->printUntilTheEndOfString();
             continue;
         }
         $lcText = strtolower($text);
         if (('true' === $lcText || 'false' === $lcText || 'null' === $lcText) && !$this->leftUsefulTokenIs([T_NS_SEPARATOR, T_AS, T_CLASS, T_EXTENDS, T_IMPLEMENTS, T_INSTANCEOF, T_INTERFACE, T_NEW, T_NS_SEPARATOR, T_PAAMAYIM_NEKUDOTAYIM, T_USE, T_TRAIT, T_INSTEADOF, T_CONST]) && !$this->rightUsefulTokenIs([T_NS_SEPARATOR, T_AS, T_CLASS, T_EXTENDS, T_IMPLEMENTS, T_INSTANCEOF, T_INTERFACE, T_NEW, T_NS_SEPARATOR, T_PAAMAYIM_NEKUDOTAYIM, T_USE, T_TRAIT, T_INSTEADOF, T_CONST]) || isset(static::$reservedWords[$lcText])) {
             $text = $lcText;
         }
         $this->appendCode($text);
     }
     return $this->code;
 }

 private function parse($src, $interestedClass = null)
 {
     $this->tokens = token_get_all($src);
     $classes = $uses = array();
     $namespace = '';
     while ($token = $this->next()) {
         if (T_NAMESPACE === $token[0]) {
             $namespace = $this->parseNamespace();
             $uses = array();
         } elseif (T_CLASS === $token[0] || T_INTERFACE === $token[0]) {
             if ('' !== $namespace) {
                 $class = $namespace . '\\' . $this->nextValue();
             } else {
                 $class = $this->nextValue();
             }
             $classes[$class] = $uses;
             if (null !== $interestedClass && $interestedClass === $class) {
                 return $classes;
             }
         } elseif (T_USE === $token[0]) {
             foreach ($this->parseUseStatement() as $useStatement) {
                 list($alias, $class) = $useStatement;
                 $uses[strtolower($alias)] = $class;
             }
         }
     }
     return $classes;
 }

 /**
  * Strips whitespace from source. Taken from composer
  * @param $source
  * @return string
  */
 private function stripWhitespace($source)
 {
     if (!function_exists('token_get_all')) {
         return $source;
     }
     $output = '';
     foreach (token_get_all($source) as $token) {
         if (is_string($token)) {
             $output .= $token;
         } elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
             // $output .= $token[1];
             $output .= str_repeat("\n", substr_count($token[1], "\n"));
         } elseif (T_WHITESPACE === $token[0]) {
             // reduce wide spaces
             $whitespace = preg_replace('{[ \\t]+}', ' ', $token[1]);
             // normalize newlines to \n
             $whitespace = preg_replace('{(?:\\r\\n|\\r|\\n)}', "\n", $whitespace);
             // trim leading spaces
             $whitespace = preg_replace('{\\n +}', "\n", $whitespace);
             $output .= $whitespace;
         } else {
             $output .= $token[1];
         }
     }
     return $output;
 }

 public function format($source)
 {
     $this->tkns = token_get_all($source);
     $this->code = '';
     while (list($index, $token) = each($this->tkns)) {
         list($id, $text) = $this->getToken($token);
         $this->ptr = $index;
         switch ($id) {
             case T_WHILE:
                 $str = $text;
                 while (list($index, $token) = each($this->tkns)) {
                     list($id, $text) = $this->getToken($token);
                     $this->ptr = $index;
                     $str .= $text;
                     if (ST_CURLY_OPEN == $id || ST_COLON == $id || ST_SEMI_COLON == $id && (ST_SEMI_COLON == $ptId || ST_CURLY_OPEN == $ptId || T_COMMENT == $ptId || T_DOC_COMMENT == $ptId)) {
                         $this->appendCode($str);
                         break;
                     } elseif (ST_SEMI_COLON == $id && !(ST_SEMI_COLON == $ptId || ST_CURLY_OPEN == $ptId || T_COMMENT == $ptId || T_DOC_COMMENT == $ptId)) {
                         $this->rtrimAndAppendCode($str);
                         break;
                     }
                 }
                 break;
             case T_WHITESPACE:
                 $this->appendCode($text);
                 break;
             default:
                 $ptId = $id;
                 $this->appendCode($text);
                 break;
         }
     }
     return $this->code;
 }

 /**
  * @param cfhCompile_CodeWriter $codeWriter
  * @param cfhCompile_Class_Interface $class
  * @param unknown_type $sourceCode
  * @param cfhCompile_ClassRegistry $classRegistry
  * @return String
  */
 public function preWrite(cfhCompile_CodeWriter $codeWriter, cfhCompile_Class_Interface $class, $sourceCode, cfhCompile_ClassRegistry $classRegistry)
 {
     if (is_null($sourceCode)) {
         return NULL;
     }
     $tokens = token_get_all('<?php ' . $sourceCode);
     $sourceCode = '';
     $lastWasString = FALSE;
     while ($token = current($tokens)) {
         $nextIsString = is_string(next($tokens));
         prev($tokens);
         if (is_string($token)) {
             $sourceCode .= $token;
             $lastWasString = TRUE;
         } else {
             list($token, $text) = $token;
             if ($token == T_WHITESPACE) {
                 if ($lastWasString === FALSE && $nextIsString === FALSE) {
                     $sourceCode .= ' ';
                 }
             } else {
                 $sourceCode .= $text;
             }
             $lastWasString = FALSE;
         }
         next($tokens);
     }
     return trim(substr($sourceCode, 5));
 }

 /**
  * Returns an array of tokens this test wants to listen for.
  *
  * @return array
  */
 public function register()
 {
     // Everyone has had a chance to figure out what forbidden functions
     // they want to check for, so now we can cache out the list.
     $this->forbiddenFunctionNames = array_keys($this->forbiddenFunctions);
     if ($this->patternMatch === true) {
         foreach ($this->forbiddenFunctionNames as $i => $name) {
             $this->forbiddenFunctionNames[$i] = '/' . $name . '/i';
         }
         return array(T_STRING);
     }
     // If we are not pattern matching, we need to work out what
     // tokens to listen for.
     $string = '<?php ';
     foreach ($this->forbiddenFunctionNames as $name) {
         $string .= $name . '();';
     }
     $register = array();
     $tokens = token_get_all($string);
     array_shift($tokens);
     foreach ($tokens as $token) {
         if (is_array($token) === true) {
             $register[] = $token[0];
         }
     }
     return array_unique($register);
 }

 public function handleTraitFile($basename, $pathname, $depth)
 {
     $traits = null;
     // The results of individual file parses are cached, since only a few
     // files will have changed and TokenisedRegularExpression is quite
     // slow. A combination of the file name and file contents hash are used,
     // since just using the datetime lead to problems with upgrading.
     $file = file_get_contents($pathname);
     $key = preg_replace('/[^a-zA-Z0-9_]/', '_', $basename) . '_' . md5($file);
     if ($data = $this->cache->load($key)) {
         $valid = isset($data['traits']) && isset($data['namespace']) && is_array($data['traits']) && is_string($data['namespace']);
         if ($valid) {
             $traits = $data['traits'];
             $namespace = $data['namespace'];
         }
     }
     if (!$traits) {
         $tokens = token_get_all($file);
         $namespace = self::get_namespace_parser()->findAll($tokens);
         if ($namespace) {
             $namespace = implode('', $namespace[0]['namespaceName']) . '\\';
         } else {
             $namespace = '';
         }
         $traits = self::get_trait_parser()->findAll($tokens);
         $cache = array('traits' => $traits, 'namespace' => $namespace);
         $this->cache->save($cache, $key);
     }
     foreach ($traits as $trait) {
         $this->traits[strtolower($namespace . $trait['traitName'])] = $pathname;
     }
 }

 public function format($source)
 {
     $this->tkns = token_get_all($source);
     $this->code = '';
     $touchedDoubleArrow = false;
     while (list($index, $token) = each($this->tkns)) {
         list($id, $text) = $this->getToken($token);
         $this->ptr = $index;
         if (T_DOUBLE_ARROW == $id) {
             $touchedDoubleArrow = true;
             $this->appendCode($text);
             continue;
         }
         if ($touchedDoubleArrow) {
             if (T_WHITESPACE == $id || T_DOC_COMMENT == $id || T_COMMENT == $id) {
                 $this->appendCode($text);
                 continue;
             }
             if (T_ARRAY === $id) {
                 $this->rtrimAndAppendCode($text);
                 $touchedDoubleArrow = false;
                 continue;
             }
             $touchedDoubleArrow = false;
         }
         $this->appendCode($text);
     }
     return $this->code;
 }

function compress_code($content)
{
    $strip_str = "";
    $tokens = token_get_all($content);
    $last_space = FALSE;
    $i = 0;
    $j = count($tokens);
    for (; $i < $j; ++$i) {
        if (is_string($tokens[$i])) {
            $last_space = FALSE;
            $strip_str .= $tokens[$i];
        } else {
            switch ($tokens[$i][0]) {
                case T_COMMENT:
                case T_DOC_COMMENT:
                case T_WHITESPACE:
                    if ($last_space) {
                        break;
                    }
                    $strip_str .= " ";
                    $last_space = TRUE;
                    break;
                default:
                    $last_space = FALSE;
                    $strip_str .= $tokens[$i][1];
            }
        }
    }
    return $strip_str;
}

 /**
  * Returns an array that contains namespace and name of the class defined in the file
  * 
  * Code losely based on http://stackoverflow.com/questions/7153000/get-class-name-from-file
  * by user http://stackoverflow.com/users/492901/netcoder
  * 
  * @param string file to anaylse
  * @return array
  */
 public static function getClassInfo($file)
 {
     $buffer = file_get_contents($file);
     $tokens = @token_get_all($buffer);
     $class = $namespace = $buffer = '';
     for ($i = 0; $i < count($tokens); $i++) {
         if ($tokens[$i][0] === T_NAMESPACE) {
             for ($j = $i + 1; $j < count($tokens); $j++) {
                 if ($tokens[$j][0] === T_STRING) {
                     $namespace .= '\\' . $tokens[$j][1];
                 } else {
                     if ($tokens[$j] === '{' || $tokens[$j] === ';') {
                         break;
                     }
                 }
             }
         }
         if ($tokens[$i][0] === T_CLASS) {
             for ($j = $i + 1; $j < count($tokens); $j++) {
                 if ($tokens[$j] === '{') {
                     if (!isset($tokens[$i + 2][1])) {
                         error_log($file . ' does not contain a valid class definition');
                         break;
                     } else {
                         $class = $tokens[$i + 2][1];
                     }
                 }
             }
         }
     }
     return array('ns' => $namespace, 'class' => $class);
 }

 /**
  * Gets the class and adds into the map
  */
 private static function get_class_name($FileName)
 {
     $Tokens = token_get_all(file_get_contents($FileName));
     $IsTestable = false;
     $IsClass = false;
     foreach ($Tokens as $Token) {
         if (is_array($Token)) {
             if (!$IsTestable && $Token[0] == T_DOC_COMMENT && strpos($Token[1], "@TestClass")) {
                 $IsTestable = true;
             }
             if ($IsTestable && $Token[0] == T_CLASS) {
                 $IsClass = true;
             } else {
                 if ($IsClass && $Token[0] == T_STRING) {
                     $ReflectionClass = new ReflectionClass($Token[1]);
                     if (count(self::get_testable_methods($ReflectionClass))) {
                         self::$Classes[$Token[1]] = new ReflectionClass($Token[1]);
                     }
                     $IsTestable = false;
                     $IsClass = false;
                 }
             }
         }
     }
 }

 /**
  * Returns the full class name for the first class in the file.
  *
  * @param string $file A PHP file path
  *
  * @return string|false Full class name if found, false otherwise
  */
 protected function findClass($file)
 {
     $class = false;
     $namespace = false;
     $tokens = token_get_all(file_get_contents($file));
     for ($i = 0, $count = count($tokens); $i < $count; $i++) {
         $token = $tokens[$i];
         if (!is_array($token)) {
             continue;
         }
         if (true === $class && T_STRING === $token[0]) {
             return $namespace . '\\' . $token[1];
         }
         if (true === $namespace && T_STRING === $token[0]) {
             $namespace = '';
             do {
                 $namespace .= $token[1];
                 $token = $tokens[++$i];
             } while ($i < $count && is_array($token) && in_array($token[0], array(T_NS_SEPARATOR, T_STRING)));
         }
         if (T_CLASS === $token[0]) {
             $class = true;
         }
         if (T_NAMESPACE === $token[0]) {
             $namespace = true;
         }
     }
     return false;
 }