Python LexToken.type方法代碼示例

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
	def _lextoken_from_html(self, html_token):
		token = LexToken()
		token.type = {
			0 : 'HTML_DOCTYPE',
			1 : 'HTML_CHARS',
			2 : 'HTML_WS',
			3 : 'HTML_STARTTAG',
			4 : 'HTML_ENDTAG',
			5 : 'HTML_EMPTYTAG',
			6 : 'HTML_COMMENT',
			7 : 'HTML_PARSEERROR',
		}[html_token['type']]
		# TODO: fix lineno/lexpos
		token.lineno = self.lineno
		token.lexpos = self.lexpos
		
		token.value = {
			'self_closing' : html_token.get('selfClosing', False),
			'name' : html_token.get('name', None),
		}
		
		if isinstance(html_token['data'], (list, tuple)):
			token.value['attrs'] = html_token['data']
			token.value['data'] = ''
			if token.value['name'].lower() in voidElements:
				token.type = 'HTML_VOID_TAG'
		else:
			token.value['data'] = html_token['data']
		
		if token.type == tokenTypes['ParseError']:
			raise SyntaxError("Got HTML Parse Error for token {}".format(html_token))
		
		return token

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
def p_error(p):
  """
  print(p.lexer.prev.lineno, p.lineno)
  if p.lexer.prev.lineno < p.lineno or p.type == "RBRACKET":
    yacc.errok()
    return
  """
      
  if p == None:
    if not restricted() and glob.g_tried_semi == False:
      t = LexToken()
      t.type = "SEMI"
      t.value = ";"
      t.lexpos = -1
      t.lineno = -1
      glob.g_lexer.push(t)
      glob.g_tried_semi = True
      yacc.errok()
    else:
      sys.stderr.write(glob.g_file + ": error: unexpected end of file\n")
    return
  else:
    glob.g_error_pre = p
    if handle_semi_error(p):
      t = LexToken()
      t.type = "SEMI"
      t.value = ";"
      t.lexpos = p.lexpos
      t.lineno = p.lineno
      #glob.g_lexer.push(t)
      #glob.g_tried_semi = True
      
      yacc.errok()
      glob.g_error = False
      return
    else:      
      glob.g_error = True
      print_err(p)
      return
      
  if glob.g_error:
    print_err(glob.g_error_pre)
    
  glob.g_error_pre = p
  glob.g_error = True
  
  try:
    line = int(p.lineno)
  except:
    line = p.lineno(1)
  
  try:
    lexdata = p.lexer.lexer.lexdata
    sline = p.lexer.lexer.lexpos
  except:
    lexdata = p.lexer.lexdata
    sline = p.lexer.lexpos
  
  sline = lexdata[sline-40:sline+1]

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def _lextoken( self, type_, value ) :
     tok = LexToken()
     tok.type = type_
     tok.value = value
     tok.lineno = self.lexer.lineno
     tok.lexpos = self.lexer.lexpos
     return tok

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def _create_token(self, type):
     token = LexToken()
     token.type = type
     token.value = ''
     token.lineno = 0
     token.lexpos = 0
     return token

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def clone_token(old_token, new_type):
   token = LexToken()
   token.type = new_type
   token.value = old_token.value
   token.lineno = old_token.lineno
   token.lexpos = old_token.lexpos
   return token

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
def _new_token(type, token):
    tok = LexToken()
    tok.type = type
    tok.value = token.value
    tok.lineno = token.lineno
    tok.lexpos = token.lexpos
    return tok

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
def new_dedent(amount, token):
    tok = LexToken()
    tok.type = "DEDENT"
    tok.value = amount
    tok.lineno = token.lineno
    tok.lexpos = token.lexpos
    return tok

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def token(self, value, ty=None):
     t = LexToken()
     t.type = ty if ty != None else value
     t.value = value
     t.lineno = -1
     t.lexpos = -1
     return t

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def _to_yacc(self, token_type, token_data):
     token = LexToken()
     token.type = token_type
     token.value = (token_type, token_data)
     token.lineno = 0 # TODO: file offset
     token.lexpos = 0
     self.__to_yacc(token)

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
def gen(code):
	for line in code:
		for item in line:
			t = LexToken()
			t.type = item[1]
			t.value = item[0]
			yield t
	yield None

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def createFunctionDefinition(self, def_token, var_token, params, val_node):
    lamToken = LexToken()
    lamToken.value = 'lambda'
    lamToken.type = 'LAMBDA'
    return LetNode(def_token, [ 
         VariableNode(var_token),
         LambdaNode(lamToken, [ Node(None, None, nodes(params)), val_node ]), 
         ])

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def newtok(tok, ttype=None):
     if tok.type != ttype and (ttype != None or tok.value != ""):
         if tok.type != None:
             push(tok)
         tok = LexToken()
         tok.type = ttype
         tok.value = ""
     return tok

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
	def _parse_chars(self, data):
		m = js_start_rx.match(data)
		
		if m is None:
			return None
		
		pretext = m.group(1)
		start_type = m.group(2)
		
		self.lexpos -= len(data)
		
		if len(pretext):
			pretext_tok = LexToken()
			pretext_tok.type = 'HTML_CHARS'
			pretext_tok.value = pretext
			pretext_tok.lineno = self.lineno - pretext.count("\n")
			pretext_tok.lexpos = self.lexpos
			self.next_tokens.append(pretext_tok)
			self.lexpos += len(pretext)
		
		start_tok = LexToken()
		start_tok.type = self.tbtype[start_type]
		start_tok.value = start_type
		start_tok.lineno = self.lineno
		start_tok.lexpos = self.lexpos
		self.next_tokens.append(start_tok)
		self.lexpos += len(start_type)
		
		js_lexer = JSLexer()
		js_lexer.input(data[m.end(2):])
		for t in js_lexer:
			t.lineno += self.lineno - 1
			t.lexpos = self.lexpos
			self.lexpos += js_lexer.lexer.lexpos
			
			if t.type in ('EXPRESSION_TERMINATOR', 'ESCAPED_TERMINATOR', 'JS_TERMINATOR'):
				if t.type != self.ttype[start_type]:
					raise SyntaxError("Expected {} but got {} in char data `{}`".format(self.ttype[start_type], t.type, data))
				self.next_tokens.append(t)
				break
			
			self.next_tokens.append(t)
		remaining_text = data[m.end(2) + js_lexer.lexer.lexpos:]
		self.lexpos += len(remaining_text)
		return remaining_text

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
    def gen_token(value, type, line, lexpos):
        t = LexToken()
        t.value = value
        t.type = type
        t.line = line
        t.lexpos = lexpos
        t.lexer = self

        return t

# 需要導入模塊: from ply.lex import LexToken [as 別名]
# 或者: from ply.lex.LexToken import type [as 別名]
 def _new_token(self, type=None, value=None, lexpos=None, lineno=None) -> LexToken:
     """
     Creates a new lexer token with the given properties.
     :return: a new lexer token with the given properties.
     """
     token = LexToken()
     token.type = type
     token.value = value
     token.lexpos = lexpos
     token.lineno = lineno