Python LexToken.lexpos方法代码示例

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
def p_error(p):
  """
  print(p.lexer.prev.lineno, p.lineno)
  if p.lexer.prev.lineno < p.lineno or p.type == "RBRACKET":
    yacc.errok()
    return
  """
      
  if p == None:
    if not restricted() and glob.g_tried_semi == False:
      t = LexToken()
      t.type = "SEMI"
      t.value = ";"
      t.lexpos = -1
      t.lineno = -1
      glob.g_lexer.push(t)
      glob.g_tried_semi = True
      yacc.errok()
    else:
      sys.stderr.write(glob.g_file + ": error: unexpected end of file\n")
    return
  else:
    glob.g_error_pre = p
    if handle_semi_error(p):
      t = LexToken()
      t.type = "SEMI"
      t.value = ";"
      t.lexpos = p.lexpos
      t.lineno = p.lineno
      #glob.g_lexer.push(t)
      #glob.g_tried_semi = True
      
      yacc.errok()
      glob.g_error = False
      return
    else:      
      glob.g_error = True
      print_err(p)
      return
      
  if glob.g_error:
    print_err(glob.g_error_pre)
    
  glob.g_error_pre = p
  glob.g_error = True
  
  try:
    line = int(p.lineno)
  except:
    line = p.lineno(1)
  
  try:
    lexdata = p.lexer.lexer.lexdata
    sline = p.lexer.lexer.lexpos
  except:
    lexdata = p.lexer.lexdata
    sline = p.lexer.lexpos
  
  sline = lexdata[sline-40:sline+1]

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def _lextoken( self, type_, value ) :
     tok = LexToken()
     tok.type = type_
     tok.value = value
     tok.lineno = self.lexer.lineno
     tok.lexpos = self.lexer.lexpos
     return tok

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def _create_token(self, type):
     token = LexToken()
     token.type = type
     token.value = ''
     token.lineno = 0
     token.lexpos = 0
     return token

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
	def _lextoken_from_html(self, html_token):
		token = LexToken()
		token.type = {
			0 : 'HTML_DOCTYPE',
			1 : 'HTML_CHARS',
			2 : 'HTML_WS',
			3 : 'HTML_STARTTAG',
			4 : 'HTML_ENDTAG',
			5 : 'HTML_EMPTYTAG',
			6 : 'HTML_COMMENT',
			7 : 'HTML_PARSEERROR',
		}[html_token['type']]
		# TODO: fix lineno/lexpos
		token.lineno = self.lineno
		token.lexpos = self.lexpos
		
		token.value = {
			'self_closing' : html_token.get('selfClosing', False),
			'name' : html_token.get('name', None),
		}
		
		if isinstance(html_token['data'], (list, tuple)):
			token.value['attrs'] = html_token['data']
			token.value['data'] = ''
			if token.value['name'].lower() in voidElements:
				token.type = 'HTML_VOID_TAG'
		else:
			token.value['data'] = html_token['data']
		
		if token.type == tokenTypes['ParseError']:
			raise SyntaxError("Got HTML Parse Error for token {}".format(html_token))
		
		return token

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
  def token(self):
    t = LexToken()
    
    c = self.cur
    if c >= len(self.str):
      return None
      
    c = self.str[c]
    if c == "\\": t.type = "BACKSLASH"
    elif c == "/": t.type = "DIVIDE"
    elif c == "[": t.type = "LSBRACKET"
    elif c == "]": t.type = "RSBRACKET"
    elif c == "*": t.type = "STAR"
    elif c == "\n" or c == "\r": t.type = "LT"
    elif re.match(r"[a-zA-Z0-9_$]+", c) != None:
      t.type = "ID_PART"
    else: t.type = "UCHAR"
    
    t.value = c
    t.lineno = 0
    t.lexpos = self.cur

    self.cur += 1
    
    print(t)
    return t

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
def _new_token(type, token):
    tok = LexToken()
    tok.type = type
    tok.value = token.value
    tok.lineno = token.lineno
    tok.lexpos = token.lexpos
    return tok

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
def new_dedent(amount, token):
    tok = LexToken()
    tok.type = "DEDENT"
    tok.value = amount
    tok.lineno = token.lineno
    tok.lexpos = token.lexpos
    return tok

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def token(self, value, ty=None):
     t = LexToken()
     t.type = ty if ty != None else value
     t.value = value
     t.lineno = -1
     t.lexpos = -1
     return t

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def _to_yacc(self, token_type, token_data):
     token = LexToken()
     token.type = token_type
     token.value = (token_type, token_data)
     token.lineno = 0 # TODO: file offset
     token.lexpos = 0
     self.__to_yacc(token)

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def clone_token(old_token, new_type):
   token = LexToken()
   token.type = new_type
   token.value = old_token.value
   token.lineno = old_token.lineno
   token.lexpos = old_token.lexpos
   return token

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
	def _parse_chars(self, data):
		m = js_start_rx.match(data)
		
		if m is None:
			return None
		
		pretext = m.group(1)
		start_type = m.group(2)
		
		self.lexpos -= len(data)
		
		if len(pretext):
			pretext_tok = LexToken()
			pretext_tok.type = 'HTML_CHARS'
			pretext_tok.value = pretext
			pretext_tok.lineno = self.lineno - pretext.count("\n")
			pretext_tok.lexpos = self.lexpos
			self.next_tokens.append(pretext_tok)
			self.lexpos += len(pretext)
		
		start_tok = LexToken()
		start_tok.type = self.tbtype[start_type]
		start_tok.value = start_type
		start_tok.lineno = self.lineno
		start_tok.lexpos = self.lexpos
		self.next_tokens.append(start_tok)
		self.lexpos += len(start_type)
		
		js_lexer = JSLexer()
		js_lexer.input(data[m.end(2):])
		for t in js_lexer:
			t.lineno += self.lineno - 1
			t.lexpos = self.lexpos
			self.lexpos += js_lexer.lexer.lexpos
			
			if t.type in ('EXPRESSION_TERMINATOR', 'ESCAPED_TERMINATOR', 'JS_TERMINATOR'):
				if t.type != self.ttype[start_type]:
					raise SyntaxError("Expected {} but got {} in char data `{}`".format(self.ttype[start_type], t.type, data))
				self.next_tokens.append(t)
				break
			
			self.next_tokens.append(t)
		remaining_text = data[m.end(2) + js_lexer.lexer.lexpos:]
		self.lexpos += len(remaining_text)
		return remaining_text

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
    def gen_token(value, type, line, lexpos):
        t = LexToken()
        t.value = value
        t.type = type
        t.line = line
        t.lexpos = lexpos
        t.lexer = self

        return t

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def _new_token(self, type=None, value=None, lexpos=None, lineno=None) -> LexToken:
     """
     Creates a new lexer token with the given properties.
     :return: a new lexer token with the given properties.
     """
     token = LexToken()
     token.type = type
     token.value = value
     token.lexpos = lexpos
     token.lineno = lineno

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def _gen_token(self, type, value='', lnum=None, position=0, lexpos=None):
     """
     Generates a LexToken with the paramaters given.
     """
     tok = LexToken()
     tok.lexer = self.lex
     tok.type = type
     tok.value = value
     tok.line_position = position
     # I think this will work...
     tok.lineno = self.lex.lineno if lnum is None else lnum
     tok.lexpos = self.lex.lexpos if lexpos is None else lexpos
     return tok

# 需要导入模块: from ply.lex import LexToken [as 别名]
# 或者: from ply.lex.LexToken import lexpos [as 别名]
 def p_error(self, p):
     # TODO
     if p:
         self._errors.append(p)
         pass # self._parser.errok()
     else:
         # hack handle eof, don't know why ply behaves this way
         from ply.lex import LexToken
         tok = LexToken()
         tok.value = self.lexer.lexdata[self.lexer.lexpos:]
         tok.lineno = self.lexer.lineno
         tok.type = 'error'
         tok.lexpos = self.lexer.lexpos
         self._parser.errok()
         return tok