Python lexer.Token类代码示例

    def output(self, handler):
        if self.__double_quotes:
            handler('"')

        Token.output(self, handler)

        if self.__double_quotes:
            handler('"')

 def output(self, handler):
     # Blocktrans output
     handler(u'{%blocktrans ');
     for p in self.params:
         p.output(handler)
         handler(u' ')
     handler(u'%}')
     Token.output(self, handler)
     handler(u'{%endblocktrans%}')

 def output(self, handler):
     # Blocktrans output
     handler(u"{%blocktrans ")
     for p in self.params:
         p.output(handler)
         handler(u" ")
     handler(u"%}")
     Token.output(self, handler)
     handler(u"{%endblocktrans%}")

    def output(self, handler):
        # Yield this node's content, or if the variable name
        # has been changed, use the modified name.
        if self.__varname:
            handler(self.__varname)

        elif self.__link_to:
            self.__link_to.output(handler)

        else:
            Token.output(self, handler)

    def output(self, handler):
        handler('<script ')
        handler(u' '.join([ u'%s%s' % (a, self.__attrs[a]) for a in self.__attrs.keys() ]))
        handler('>')

        if not self.is_external:
            handler('//<![CDATA[\n')

        Token.output(self, handler)

        if not self.is_external:
            handler(u'//]]>\n')

        handler(u'</script>')

def fix_whitespace_bug(js_node):
    """
    Fixes the following case in js code:
        <script type="text/javascript"> if {  {% if test %} ... {% endif %} } </script>
    The lexer above would remove the space between the first '{' and '{%'. This collision
    would make Django think it's the start of a variable.
    """
    # For every scope (starting with '{')
    for scope in js_node.child_nodes_of_class(JavascriptScope):
        # Look if the first child inside this scope also renders to a '{'
        if scope.children and scope.children[0].output_as_string()[0:1] == '{':
            # If so, insert a whitespace token in between.
            space = Token(name='required-whitespace')
            space.children = [' ']
            scope.children.insert(0, space)

def compile_javascript_string(js_string, context, path=''):
    """
    Compile JS code (can be used for external javascript files)
    """
    # First, create a tree to begin with
    tree = Token(name='root', line=1, column=1, path=path)
    tree.children = [ js_string ]

    # Tokenize
    tokenize(tree, __JS_STATES, Token)

    # Compile
    _compile(tree, context)

    # Output
    return tree.output_as_string()

def compile_html_string(html_string, path=''):
    """
    Compile a html string
    """
    # First, create a tree to begin with
    tree = Token(name='root', line=1, column=1, path=path)
    tree.children = [ html_string ]

    # Tokenize
    tokenize(tree, __HTML_STATES, Token)

    from template_preprocessor.core.context import Context
    context = Context(path)
    _process_html_tree(tree, context)

    # Output
    return tree.output_as_string()

def compile_javascript_string(js_string, context, path=''):
    """
    Compile JS code (can be used for external javascript files)
    """

    # First, create a tree to begin with
    tree = Token(name='root', line=1, column=1, path=path)
    tree.children = [ js_string ]

    # Tokenize
    with context.time_operation("tokenize string"):
        tokenize(tree, __JS_STATES, Token)

    # Compile
    _compile(tree, context, already_minified=path.endswith(".min.js"))

    # Output
    return tree.output_as_string()

def compile_html_string(html_string, path=""):
    """
    Compile a html string
    """
    # First, create a tree to begin with
    tree = Token(name="root", line=1, column=1, path=path)
    tree.children = [html_string]

    # Tokenize
    tokenize(tree, __HTML_STATES, [Token])

    from template_preprocessor.core.django_processor import PreProcessSettings

    options = PreProcessSettings()
    _process_html_tree(tree, options)

    # Output
    return tree.output_as_string()

def compile_css_string(css_string, context, path='', url=None):
    """
    Compile CSS code
    """
    # First, create a tree to begin with
    tree = Token(name='root', line=1, column=1, path=path)
    tree.children = [ css_string ]

    # Tokenize
    tokenize(tree, __CSS_STATES, Token)
    _add_css_parser_extensions(tree)

    # Rewrite url() in external css files
    if url:
        _rewrite_urls(tree, url)

    # Compile
    _compress_css_whitespace(tree)

    # Output
    return u''.join([o for o in tree.output_as_string() ])

def compile_javascript_string(js_string, path=''):
    """
    Compile JS code (can be used for external javascript files)
    """
    # First, create a tree to begin with
    tree = Token(name='root', line=1, column=1, path=path)
    tree.children = [ js_string ]

    # Tokenize
    tokenize(tree, __JS_STATES, [Token] )

    # Compile
    _add_javascript_parser_extensions(tree)
    _validate_javascript(tree)
    _compress_javascript_whitespace(tree)
    _minify_variable_names(tree)

    fix_whitespace_bug(tree)

    # Output
    return tree.output_as_string()

 def init_extension(self):
     self.__varname = Token.output_as_string(self, True)

def parse(source_code, path, context, main_template=False):
    """
    Parse the code.
    - source_code: string
    - path: for attaching meta information to the tree.
    - context: preprocess context (holding the settings/dependecies/warnings, ...)
    - main_template: False for includes/extended templates. True for the
                     original path that was called.
    """
    # To start, create the root node of a tree.
    tree = Token(name="root", line=1, column=1, path=path)
    tree.children = [source_code]

    # Lex Django tags
    tokenize(tree, __DJANGO_STATES, Token)

    # Phase I: add parser extensions
    _add_parser_extensions(tree)

    # Phase II: process inline tags
    _process_inline_tags(tree)

    # Phase III: create recursive structure for block level tags.
    nest_block_level_elements(tree, __DJANGO_BLOCK_ELEMENTS, DjangoTag, lambda c: c.tagname)

    # === Actions ===

    if main_template:
        _find_first_level_dependencies(tree, context)

    # Extend parent template and process includes
    tree = _process_extends(tree, context)  # NOTE: this returns a new tree!
    _preprocess_includes(tree, context)
    _preprocess_decorate_tags(tree, context)

    # Following actions only need to be applied if this is the 'main' tree.
    # It does not make sense to apply it on every include, and then again
    # on the complete tree.
    if main_template:

        _update_preprocess_settings(tree, context)
        options = context.options

        # Remember translations in context (form PO-file generation)
        remember_gettext_entries(tree, context)

        # Do translations
        if options.preprocess_translations:
            _preprocess_trans_tags(tree)

        # Reverse URLS
        if options.preprocess_urls:
            _preprocess_urls(tree)

        # Do variable lookups
        if options.preprocess_variables:
            sites_enabled = "django.contrib.sites" in settings.INSTALLED_APPS

            _preprocess_variables(
                tree,
                {"MEDIA_URL": getattr(settings, "MEDIA_URL", ""), "STATIC_URL": getattr(settings, "STATIC_URL", "")},
            )
            if sites_enabled:
                from django.contrib.sites.models import Site

                try:
                    # Don't preprocess anything when we don't have a Site
                    # instance yet.
                    site = Site.objects.get_current()
                    _preprocess_variables(
                        tree,
                        {"SITE_DOMAIN": site.domain, "SITE_NAME": site.name, "SITE_URL": "http://%s" % site.domain},
                    )
                except Site.DoesNotExist, e:
                    pass

        # Don't output {% block %} tags in the compiled file.
        if options.remove_block_tags:
            tree.collapse_nodes_of_class(DjangoBlockTag)

        # Preprocess {% callmacro %} tags
        if options.preprocess_macros:
            _preprocess_macros(tree)

        # Group all {% load %} statements
        if options.merge_all_load_tags:
            _group_all_loads(tree)

        # Preprocessable tags
        if options.execute_preprocessable_tags:
            _execute_preprocessable_tags(tree)

        # HTML compiler
        if options.is_html:
            compile_html(tree, context)