Python tokeniter.tokens函数代码示例

 def html_selection(self, cursor):
     """Return HTML for the cursor's selection."""
     d = cursor.document()
     start = d.findBlock(cursor.selectionStart())
     startpos = cursor.selectionStart() - start.position()
     end = d.findBlock(cursor.selectionEnd())
     endpos = cursor.selectionEnd() - end.position()
     
     html = []
     # first block, skip tokens before selection
     block = start
     source = iter(tokeniter.tokens(block))
     for t in source:
         if t.end > startpos:
             startslice = max(0, startpos - t.pos)
             endslice = None
             if block == end and t.end > endpos:
                 endslice = endpos - t.pos
             html.append(self.html_for_token(t[startslice:endslice], type(t)))
             break
     while block != end:
         html.extend(map(self.html_for_token, source))
         html.append('\n')
         block = block.next()
         source = iter(tokeniter.tokens(block))
     # last block, go to end of selection
     for t in source:
         if t.end > endpos:
             if t.pos < endpos:
                 html.append(self.html_for_token(t[:endpos-t.pos], type(t)))
             break
         html.append(self.html_for_token(t))
     return self.html_wrapper("".join(html))

def change_indent(cursor, direction):
    """Changes the indent in the desired direction (-1 for left and +1 for right).
    
    Returns True if the indent operation was applied.
    The cursor may contain a selection.
    
    """
    # get some variables from the document
    indent_vars = indent_variables(cursor.document())
    
    blocks = list(cursortools.blocks(cursor))
    block = blocks[0]
    pos = cursor.selectionStart() - block.position()
    token = tokeniter.tokens(block)[0] if tokeniter.tokens(block) else None
    if cursor.hasSelection() or pos == 0 or (token and isinstance(token, ly.lex.Space) and token.end >= pos):
        # decrease the indent
        state = tokeniter.state(block)
        current_indent = get_indent(block)
        new_indent = current_indent + direction * indent_vars['indent-width']
        if state.mode() in ('lilypond', 'scheme'):
            computed_indent = compute_indent(block)
            if cmp(computed_indent, new_indent) == direction:
                new_indent = computed_indent
        diff = new_indent - current_indent
        with cursortools.compress_undo(cursor):
            for block in blocks:
                set_indent(block, get_indent(block) + diff)
        return True

def reformat(cursor):
    """Reformats the selection or the whole document, adjusting the whitespace."""
    def newlinebefore(t):
        editor.insertText(tokeniter.cursor(block, t, end=0), '\n')
    
    def newlineafter(t):
        editor.insertText(tokeniter.cursor(block, t, start=len(t)), '\n')
    
    indent_vars = indent.indent_variables(cursor.document())
    
    with cursortools.compress_undo(cursor):
        indent.re_indent(cursor)
        with cursortools.Editor() as editor:
            for block in get_blocks(cursor):
                
                denters = []
                tokens = tokeniter.tokens(block)
                
                nonspace_index = -1
                for i, t in enumerate(tokens):
                    if isinstance(t, ly.lex.Indent) and t in ('{', '<<'):
                        denters.append(i)
                    elif isinstance(t, ly.lex.Dedent) and t in ('}', '>>'):
                        if denters:
                            denters.pop()
                        elif nonspace_index != -1:
                            newlinebefore(t)
                    elif not isinstance(t, ly.lex.Space):
                        nonspace_index = i
                for i in denters:
                    if i < nonspace_index:
                        newlineafter(tokens[i])
                    
                # TODO: wrap long lines
        
        indent.re_indent(cursor)
        
        with cursortools.Editor() as editor:
            for block in get_blocks(cursor):
                tokens = tokeniter.tokens(block)
                if (len(tokens) == 2
                    and isinstance(tokens[0], ly.lex.Space)
                    and isinstance(tokens[1], (
                        ly.lex.lilypond.LineComment,
                        ly.lex.scheme.LineComment))
                    and len(tokens[1]) > 2
                    and len(set(tokens[1][:3])) == 1):
                    # move commented lines with more than 2 comment characters
                    # to column 0
                    editor.removeSelectedText(tokeniter.cursor(block, tokens[0]))
                else:
                    # remove trialing whitespace
                    for t in tokens[::-1]:
                        if isinstance(t, ly.lex.Space):
                            editor.removeSelectedText(tokeniter.cursor(block, t))
                        else:
                            break

def open_file_at_cursor(cursor, mainwin):
    """Opens the filename mentioned at the text cursor."""
    # take either the selection or the include-args found by ly.parse
    if cursor.hasSelection():
        fnames = [cursor.selection().toPlainText()]
    else:
        fnames = list(ly.parse.includeargs(iter(tokeniter.tokens(cursor.block()))))
    
    # detemine search path: doc dir and other include path names
    filename = cursor.document().url().toLocalFile()
    if filename:
        path = [os.path.dirname(filename)]
    else:
        path = []
    path.extend(documentinfo.info(cursor.document()).includepath())
    
    # load all docs, trying all include paths
    d = None
    for f in fnames:
        for p in path:
            name = os.path.normpath(os.path.join(p, f))
            if os.access(name, os.R_OK):
                d = mainwin.openUrl(QUrl.fromLocalFile(name))
                break
    if d:
        mainwin.setCurrentDocument(d, True)

 def fold_events(self, block):
     """Provides folding information by looking at indent/dedent tokens."""
     for t in tokeniter.tokens(block):
         if isinstance(t, ly.lex.Indent):
             yield widgets.folding.START
         elif isinstance(t, ly.lex.Dedent):
             yield widgets.folding.STOP

 def actionTriggered(self, name):
     # convert arpeggio_normal to arpeggioNormal, etc.
     name = _arpeggioTypes[name]
     cursor = self.mainwindow().textCursor()
     # which arpeggio type is last used?
     lastused = '\\arpeggioNormal'
     types = set(_arpeggioTypes.values())
     block = cursor.block()
     while block.isValid():
         s = types.intersection(tokeniter.tokens(block))
         if s:
             lastused = s.pop()
             break
         block = block.previous()
     # where to insert
     c = lydocument.cursor(cursor)
     c.select_end_of_block()
     source = lydocument.Source(c, True, ly.document.OUTSIDE, True)
     with cursortools.compress_undo(cursor):
         for p in ly.rhythm.music_tokens(source):
             c = source.cursor(p[-1], start=len(p[-1]))
             c.insertText('\\arpeggio')
             if name != lastused:
                 cursortools.strip_indent(c)
                 indent = c.block().text()[:c.position()-c.block().position()]
                 c.insertText(name + '\n' + indent)
             # just pick the first place
             return

 def actionTriggered(self, name):
     # convert arpeggio_normal to arpeggioNormal, etc.
     name = _arpeggioTypes[name]
     cursor = self.mainwindow().textCursor()
     # which arpeggio type is last used?
     lastused = '\\arpeggioNormal'
     types = set(_arpeggioTypes.values())
     block = cursor.block()
     while block.isValid():
         s = types.intersection(tokeniter.tokens(block))
         if s:
             lastused = s.pop()
             break
         block = block.previous()
     # where to insert
     source = tokeniter.Source.from_cursor(cursor, True, -1)
     with cursortools.compress_undo(cursor):
         for p in music.music_items(source, tokens=source.tokens):
             c = source.cursor(p[-1], start=len(p[-1]))
             c.insertText('\\arpeggio')
             if name != lastused:
                 cursortools.strip_indent(c)
                 import indent
                 indent.insert_text(c, name + '\n')
             # just pick the first place
             return

def back(cursor):
    """Yields per-block token iters in backward direction from the cursor."""
    yield reversed(tokeniter.partition(cursor).left)
    block = cursor.block()
    while block.previous().isValid():
        block = block.previous()
        yield reversed(tokeniter.tokens(block))

def tokens(cursor):
    """Yield the tokens tuple for every block from the beginning of the document until the cursor."""
    end = cursor.block()
    block = cursor.document().firstBlock()
    while block < end:
        yield tokeniter.tokens(block)
        block = block.next()

def cut_assign(cursor):
    """Cuts selected text and assigns it to a LilyPond variable."""
    # ask the variable name
    name = inputdialog.getText(None, _("Cut and Assign"), _(
        "Please enter the name for the variable to assign the selected "
        "text to:"), regexp="[A-Za-z]+")
    if not name:
        return
    
    cursortools.strip_selection(cursor)
    
    # determine state at cursor
    block = cursortools.block(cursor)
    state = tokeniter.state(block)
    for t in tokeniter.partition(cursor).left:
        state.follow(t)
    
    mode = ""
    for p in state.parsers():
        if isinstance(p, ly.lex.lilypond.ParseInputMode):
            if isinstance(p, ly.lex.lilypond.ParseLyricMode):
                mode = " \\lyricmode"
            elif isinstance(p, ly.lex.lilypond.ParseChordMode):
                mode = " \\chordmode"
            elif isinstance(p, ly.lex.lilypond.ParseFigureMode):
                mode = " \\figuremode"
            elif isinstance(p, ly.lex.lilypond.ParseDrumMode):
                mode = " \\drummode"
            break

    # find insertion place:
    found = False
    while block.previous().isValid():
        block = block.previous()
        state = tokeniter.state(block)
        if isinstance(state.parser(), ly.lex.lilypond.ParseGlobal):
            found = True
            break
        tokens = tokeniter.tokens(block)
        for t in tokens:
            if isinstance(t, ly.lex.lilypond.Name):
                found = True
                break
            elif not isinstance(t, (ly.lex.Space, ly.lex.Comment)):
                break
        if found:
            break
    insert = QTextCursor(block)
    text = cursor.selection().toPlainText()
    space = '\n' if '\n' in text else ' '
    text = ''.join((name, ' =', mode, ' {', space, text, space, '}\n\n'))
    with cursortools.compress_undo(cursor):
        cursor.insertText('\\' + name)
        pos = insert.selectionStart()
        insert.insertText(text)
    if metainfo.info(cursor.document()).auto_indent:
        insert.setPosition(pos, QTextCursor.KeepAnchor)
        with cursortools.compress_undo(insert, True):
            indent.re_indent(insert)

def indentable(cursor):
    """Returns True if the cursor is at a dedent token and running the auto-indenter makes sense."""
    block = cursor.block()
    pos = cursor.position() - block.position()
    for token in tokeniter.tokens(block):
        if token.end >= pos:
            return isinstance(token, (ly.lex.Dedent, ly.lex.BlockCommentEnd))
        elif not isinstance(token, (ly.lex.Space, ly.lex.Dedent)):
            return

def get_indent(block):
    """Returns the indent of the given block."""
    
    # get some variables from the document
    indent_vars = indent_variables(block.document())
    
    tokens = tokeniter.tokens(block)
    if tokens and isinstance(tokens[0], ly.lex.Space):
        return column_position(tokens[0], tabwidth = indent_vars['tab-width'])
    return 0

def names(cursor):
    """Harvests names from assignments until the cursor."""
    end = cursor.block()
    block = cursor.document().firstBlock()
    while block.isValid() and block != end:
        for t in tokeniter.tokens(block)[:2]:
            if type(t) is ly.lex.lilypond.Name:
                yield t
                break
        block = block.next()

def get_definition(cursor):
    block = cursor.block()
    while block.isValid():
        state = tokeniter.state(block)
        if isinstance(state.parser(), ly.lex.lilypond.ParseGlobal):
            for t in tokeniter.tokens(block)[:2]:
                if type(t) is ly.lex.lilypond.Name:
                    return t[:]
                elif isinstance(t, ly.lex.lilypond.Keyword) and t == '\\score':
                    return '\\score'
        block = block.previous()

 def globalStaffSize(self, default=20):
     """Returns the global staff size, if set, else the default value."""
     for block in cursortools.all_blocks(self.document()):
         tokens = tokeniter.tokens(block)
         try:
             i = tokens.index('set-global-staff-size')
         except ValueError:
             continue
         try:
             return int(tokens[i+2], 10)
         except (IndexError, ValueError):
             pass
     return default