PHP Twig_Environment::tokenize方法代码示例

 public function testEmptyConstant()
 {
     $stream = $this->environment->parse($this->environment->tokenize('{{ constant("ExtensionTest::SOME_EMPTY_VALUE") }}', 'index'));
     $node = $stream->getNode('body')->getNode(0);
     $this->assertEquals(Twig_Node_Print::class, get_class($node));
     $this->assertEquals(ExtensionTest::SOME_EMPTY_VALUE, $node->getNode('expr')->getAttribute('value'));
 }

 /**
  * @param EmailTemplate                  $value
  * @param Constraint|EmailTemplateSyntax $constraint
  */
 public function validate($value, Constraint $constraint)
 {
     // prepare templates to be validated
     $itemsToValidate = [['field' => 'subject', 'locale' => null, 'template' => $value->getSubject()], ['field' => 'content', 'locale' => null, 'template' => $value->getContent()]];
     $translations = $value->getTranslations();
     foreach ($translations as $trans) {
         if (in_array($trans->getField(), ['subject', 'content'])) {
             $itemsToValidate[] = ['field' => $trans->getField(), 'locale' => $trans->getLocale(), 'template' => $trans->getContent()];
         }
     }
     /** @var \Twig_Extension_Sandbox $sandbox */
     $sandbox = $this->twig->getExtension('sandbox');
     $sandbox->enableSandbox();
     // validate templates' syntax
     $errors = [];
     foreach ($itemsToValidate as &$item) {
         try {
             $this->twig->parse($this->twig->tokenize($item['template']));
         } catch (\Twig_Error_Syntax $e) {
             $errors[] = ['field' => $item['field'], 'locale' => $item['locale'], 'error' => $e->getMessage()];
         }
     }
     $sandbox->disableSandbox();
     // add violations for found errors
     if (!empty($errors)) {
         foreach ($errors as $error) {
             $this->context->addViolation($constraint->message, ['{{ field }}' => $this->getFieldLabel(ClassUtils::getClass($value), $error['field']), '{{ locale }}' => $this->getLocaleName($error['locale']), '{{ error }}' => $error['error']]);
         }
     }
 }

 function visitFile(\SplFileInfo $file, MessageCatalogue $catalogue)
 {
     if ($file->getExtension() == 'twital' && ($adapter = $this->twitalLoader->getSourceAdapter((string) $file))) {
         $source = $this->twitalLoader->getTwital()->compile($adapter, file_get_contents((string) $file));
         $ast = $this->twig->parse($this->twig->tokenize($source, (string) $file));
         $this->visitTwigFile($file, $catalogue, $ast);
     }
 }

 /**
  * {@inheritDoc}
  */
 public function getPlaces($layoutSrc)
 {
     $tokenStream = $this->twig->tokenize($this->twig->getLoader()->getSource($layoutSrc));
     $collector = new PlaceHolderNodeCollector();
     $traverser = new \Twig_NodeTraverser($this->twig, array($collector));
     $traverser->traverse($this->twig->parse($tokenStream));
     return $collector->getCollectedNames();
 }

 protected function extractTemplate($template, MessageCatalogue $catalogue)
 {
     $visitor = $this->twig->getExtension('translator')->getTranslationNodeVisitor();
     $visitor->enable();
     $this->twig->parse($this->twig->tokenize($template));
     foreach ($visitor->getMessages() as $message) {
         $catalogue->set(trim($message[0]), $this->prefix . trim($message[0]), $message[1] ? $message[1] : $this->defaultDomain);
     }
     $visitor->disable();
 }

 /**
  * {@inheritDoc}
  */
 public function extract($directory, MessageCatalogue $catalogue)
 {
     // load any existing translation files
     $finder = new Finder();
     $files = $finder->files()->name('*.twig')->in($directory);
     foreach ($files as $file) {
         $tree = $this->twig->parse($this->twig->tokenize(file_get_contents($file->getPathname())));
         $this->crawlNode($tree, $catalogue);
     }
 }

 protected function extractTemplate($template, MessageCatalogue $catalogue)
 {
     $visitor = $this->twig->getExtension('Symfony\\Bridge\\Twig\\Extension\\TranslationExtension')->getTranslationNodeVisitor();
     $visitor->enable();
     $this->twig->parse($this->twig->tokenize(new \Twig_Source($template, '')));
     foreach ($visitor->getMessages() as $message) {
         $catalogue->set(trim($message[0]), $this->prefix . trim($message[0]), $message[1] ?: $this->defaultDomain);
     }
     $visitor->disable();
 }

 /**
  * @dataProvider getTestsForString
  */
 public function testStringExpression($template, $expected)
 {
     $env = new Twig_Environment(new Twig_Loader_String(), array('cache' => false, 'autoescape' => false, 'optimizations' => 0));
     $stream = $env->tokenize($template, 'index');
     $parser = new Twig_Parser($env);
     $this->assertEquals($expected, $parser->parse($stream)->getNode('body')->getNode(0)->getNode('expr'));
 }

    /**
     * @expectedException Twig_Error_Syntax
     * @dataProvider getFailingTestsForArray
     */
    public function testArraySyntaxError($template)
    {
        $env = new Twig_Environment(new Twig_Loader_String(), array('cache' => false, 'autoescape' => false));
        $parser = new Twig_Parser($env);

        $parser->parse($env->tokenize($template, 'index'));
    }

 /**
  * @dataProvider getEscapingTemplates
  */
 public function testEscaping($template, $mustBeEscaped)
 {
     $twig = new \Twig_Environment(null, array('debug' => true, 'cache' => false, 'autoescape' => true, 'optimizations' => 0));
     $twig->addExtension(new RoutingExtension($this->getMock('Symfony\\Component\\Routing\\Generator\\UrlGeneratorInterface')));
     $nodes = $twig->parse($twig->tokenize($template));
     $this->assertSame($mustBeEscaped, $nodes->getNode('body')->getNode(0)->getNode('expr') instanceof \Twig_Node_Expression_Filter);
 }

 /**
  * @group legacy
  */
 public function testLegacyTokenizeSignature()
 {
     $env = new Twig_Environment();
     $stream = $env->tokenize('{{ foo }}', 'foo');
     $this->assertEquals('{{ foo }}', $stream->getSource());
     $this->assertEquals('foo', $stream->getFilename());
 }

 /**
  * Tests the escaping
  *
  * @dataProvider providerTestEscaping
  */
 public function testEscaping($template, $expected)
 {
     $twig = new \Twig_Environment(NULL, array('debug' => TRUE, 'cache' => FALSE, 'autoescape' => TRUE, 'optimizations' => 0));
     $twig->addExtension((new TwigExtension())->setGenerators($this->getMock('Drupal\\Core\\Routing\\UrlGeneratorInterface')));
     $nodes = $twig->parse($twig->tokenize($template));
     $this->assertSame($expected, $nodes->getNode('body')->getNode(0)->getNode('expr') instanceof \Twig_Node_Expression_Filter);
 }

 /**
  * @dataProvider fixtureProvider
  */
 public function testCompile($source, $expected)
 {
     $twig = new \Twig_Environment($this->getMockBuilder('Twig_LoaderInterface')->getMock(), ['cache' => false, 'autoescape' => false, 'optimizations' => 0]);
     $twig->addExtension(new ElixirExtension('public', 'build'));
     $nodes = $twig->parse($twig->tokenize($source));
     $this->assertEquals($expected, $nodes->getNode('body')->getNode(0));
 }

 /**
  * @dataProvider getTests
  */
 public function testIntegration($file, $test, $message, $templates)
 {
     $loader = new Twig_Loader_Array($templates);
     $twig = new Twig_Environment($loader, array('trim_blocks' => true, 'cache' => false));
     $twig->addExtension(new Twig_Extension_Escaper());
     $twig->addExtension(new TestExtension());
     try {
         $template = $twig->loadTemplate('index.twig');
     } catch (Twig_SyntaxError $e) {
         $e->setFilename($file);
         throw $e;
     } catch (Exception $e) {
         throw new Twig_Error($e->getMessage() . ' (in ' . $file . ')');
     }
     preg_match_all('/--DATA--(.*?)--EXPECT--(.*?)(?=\\-\\-DATA\\-\\-|$)/s', $test, $matches, PREG_SET_ORDER);
     foreach ($matches as $match) {
         $output = trim($template->render(eval($match[1] . ';')), "\n ");
         $expected = trim($match[2], "\n ");
         if ($expected != $output) {
             echo 'Compiled template that failed:';
             foreach (array_keys($templates) as $name) {
                 echo "Template: {$name}\n";
                 $source = $loader->getSource($name);
                 echo $twig->compile($twig->parse($twig->tokenize($source, $name)));
             }
         }
         $this->assertEquals($expected, $output, $message . ' (in ' . $file . ')');
     }
 }

 /**
  * @param string $path
  *
  * @return array|Word[]
  */
 public function parseFile($path)
 {
     $words = array();
     $twig = new \Twig_Environment();
     // TODO should it be configured dynamically?
     $lines = file($path);
     if (count($lines) <= 0) {
         // TODO handle and log error
         return array();
     }
     $tokenStream = $twig->tokenize(implode('', $lines));
     while (!$tokenStream->isEOF()) {
         $token = $tokenStream->next();
         if ($token->getType() === \Twig_Token::TEXT_TYPE) {
             $lineNumber = $token->getLine();
             //split line by line
             foreach (preg_split("/((\r?\n)|(\r\n?))/", $token->getValue()) as $line) {
                 $parts = array();
                 preg_match_all('/(([[:alpha:]]+[\'-])*[[:alpha:]]+\'?)/u', strip_tags($line), $parts, PREG_SET_ORDER);
                 foreach ($parts as $part) {
                     $words[] = new Word(trim($part[0]), $path, $lines[$lineNumber - 1], $lineNumber);
                 }
                 ++$lineNumber;
             }
         }
     }
     return $words;
 }