Golang Tokenizer.Scan方法代码示例

// parsePkTuple parese one pk tuple.
func parsePkTuple(tokenizer *sqlparser.Tokenizer) (pkTuple *sqlparser.Node, err error) {
	// pkTuple is a list of pk value Nodes
	pkTuple = sqlparser.NewSimpleParseNode(sqlparser.NODE_LIST, "")
	// start scanning the list
	for tempNode := tokenizer.Scan(); tempNode.Type != ')'; tempNode = tokenizer.Scan() {
		switch tempNode.Type {
		case '-':
			// handle negative numbers
			t2 := tokenizer.Scan()
			if t2.Type != sqlparser.NUMBER {
				return nil, fmt.Errorf("expecing number after '-'")
			}
			t2.Value = append(tempNode.Value, t2.Value...)
			pkTuple.Push(t2)
		case sqlparser.ID, sqlparser.NUMBER, sqlparser.NULL:
			pkTuple.Push(tempNode)
		case sqlparser.STRING:
			b := tempNode.Value
			decoded := make([]byte, base64.StdEncoding.DecodedLen(len(b)))
			numDecoded, err := base64.StdEncoding.Decode(decoded, b)
			if err != nil {
				return nil, err
			}
			tempNode.Value = decoded[:numDecoded]
			pkTuple.Push(tempNode)
		default:
			return nil, fmt.Errorf("unexpected token: '%v'", string(tempNode.Value))
		}
	}
	return pkTuple, nil
}

//This parese a paricular pk tuple.
func parsePkTuple(tokenizer *parser.Tokenizer, autoincIdPtr *uint64) (pkTuple *parser.Node) {
	//pkTuple is a list of pk value Nodes
	pkTuple = parser.NewSimpleParseNode(parser.NODE_LIST, "")
	autoincId := *autoincIdPtr
	//start scanning the list
	for tempNode := tokenizer.Scan(); tempNode.Type != ')'; tempNode = tokenizer.Scan() {
		switch tempNode.Type {
		case parser.NULL:
			pkTuple.Push(parser.NewParseNode(parser.NUMBER, []byte(strconv.FormatUint(autoincId, 10))))
			autoincId++
		case '-':
			//handle negative numbers
			t2 := tokenizer.Scan()
			if t2.Type != parser.NUMBER {
				panic(NewBinlogParseError(CODE_ERROR, "Illegal stream comment construct, - followed by a non-number"))
			}
			t2.Value = append(tempNode.Value, t2.Value...)
			pkTuple.Push(t2)
		case parser.ID, parser.NUMBER:
			pkTuple.Push(tempNode)
		case parser.STRING:
			b := tempNode.Value
			decoded := make([]byte, base64.StdEncoding.DecodedLen(len(b)))
			numDecoded, err := base64.StdEncoding.Decode(decoded, b)
			if err != nil {
				panic(NewBinlogParseError(CODE_ERROR, "Error in base64 decoding pkValue"))
			}
			tempNode.Value = decoded[:numDecoded]
			pkTuple.Push(tempNode)
		default:
			panic(NewBinlogParseError(EVENT_ERROR, fmt.Sprintf("Error in parsing stream comment: illegal node type %v %v", tempNode.Type, string(tempNode.Value))))
		}
	}
	return pkTuple
}

// parsePkTuple parese one pk tuple.
func parsePkTuple(tokenizer *sqlparser.Tokenizer) (tuple sqlparser.ValTuple, err error) {
	// start scanning the list
	for typ, val := tokenizer.Scan(); typ != ')'; typ, val = tokenizer.Scan() {
		switch typ {
		case '-':
			// handle negative numbers
			typ2, val2 := tokenizer.Scan()
			if typ2 != sqlparser.NUMBER {
				return nil, fmt.Errorf("expecing number after '-'")
			}
			num := append(sqlparser.NumVal("-"), val2...)
			tuple = append(tuple, num)
		case sqlparser.NUMBER:
			tuple = append(tuple, sqlparser.NumVal(val))
		case sqlparser.NULL:
			tuple = append(tuple, new(sqlparser.NullVal))
		case sqlparser.STRING:
			decoded := make([]byte, base64.StdEncoding.DecodedLen(len(val)))
			numDecoded, err := base64.StdEncoding.Decode(decoded, val)
			if err != nil {
				return nil, err
			}
			tuple = append(tuple, sqlparser.StrVal(decoded[:numDecoded]))
		default:
			return nil, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
		}
	}
	return tuple, nil
}

func parsePkNames(tokenizer *sqlparser.Tokenizer) (columns []string, err error) {
	if typ, _ := tokenizer.Scan(); typ != '(' {
		return nil, fmt.Errorf("expecting '('")
	}
	for typ, val := tokenizer.Scan(); typ != ')'; typ, val = tokenizer.Scan() {
		switch typ {
		case sqlparser.ID:
			columns = append(columns, string(val))
		default:
			return nil, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
		}
	}
	return columns, nil
}

func parsePkNames(tokenizer *sqlparser.Tokenizer) (columns []string, err error) {
	tempNode := tokenizer.Scan()
	if tempNode.Type != '(' {
		return nil, fmt.Errorf("expecting '('")
	}
	for tempNode := tokenizer.Scan(); tempNode.Type != ')'; tempNode = tokenizer.Scan() {
		switch tempNode.Type {
		case sqlparser.ID:
			columns = append(columns, string(tempNode.Value))
		default:
			return nil, fmt.Errorf("unexpected token: '%v'", string(tempNode.Value))
		}
	}
	return columns, nil
}

// parsePkTuple parses something like (null 1 'bmFtZQ==' )
func parsePkTuple(tokenizer *sqlparser.Tokenizer, insertid int64, fields []mproto.Field) ([]sqltypes.Value, int64, error) {
	var result []sqltypes.Value

	// start scanning the list
	index := 0
	for typ, val := tokenizer.Scan(); typ != ')'; typ, val = tokenizer.Scan() {
		if index >= len(fields) {
			return nil, insertid, fmt.Errorf("length mismatch in values")
		}

		switch typ {
		case '-':
			// handle negative numbers
			typ2, val2 := tokenizer.Scan()
			if typ2 != sqlparser.NUMBER {
				return nil, insertid, fmt.Errorf("expecting number after '-'")
			}

			// check value
			fullVal := append([]byte{'-'}, val2...)
			if _, err := strconv.ParseInt(string(fullVal), 0, 64); err != nil {
				return nil, insertid, err
			}

			// update type
			switch fields[index].Type {
			case mproto.VT_DECIMAL:
				// we haven't updated the type yet
				fields[index].Type = mproto.VT_LONGLONG
			case mproto.VT_LONGLONG:
				// nothing to do there
			default:
				// we already set this to something incompatible!
				return nil, insertid, fmt.Errorf("incompatible negative number field with type %v", fields[index].Type)
			}

			// update value
			result = append(result, sqltypes.MakeNumeric(fullVal))

		case sqlparser.NUMBER:
			// check value
			if _, err := strconv.ParseUint(string(val), 0, 64); err != nil {
				return nil, insertid, err
			}

			// update type
			switch fields[index].Type {
			case mproto.VT_DECIMAL:
				// we haven't updated the type yet
				fields[index].Type = mproto.VT_LONGLONG
			case mproto.VT_LONGLONG:
				// nothing to do there
			default:
				// we already set this to something incompatible!
				return nil, insertid, fmt.Errorf("incompatible number field with type %v", fields[index].Type)
			}

			// update value
			result = append(result, sqltypes.MakeNumeric(val))
		case sqlparser.NULL:
			// update type
			switch fields[index].Type {
			case mproto.VT_DECIMAL:
				// we haven't updated the type yet
				fields[index].Type = mproto.VT_LONGLONG
			case mproto.VT_LONGLONG:
				// nothing to do there
			default:
				// we already set this to something incompatible!
				return nil, insertid, fmt.Errorf("incompatible auto-increment field with type %v", fields[index].Type)
			}

			// update value
			result = append(result, sqltypes.MakeNumeric(strconv.AppendInt(nil, insertid, 10)))
			insertid++
		case sqlparser.STRING:
			// update type
			switch fields[index].Type {
			case mproto.VT_DECIMAL:
				// we haven't updated the type yet
				fields[index].Type = mproto.VT_VAR_STRING
			case mproto.VT_VAR_STRING:
				// nothing to do there
			default:
				// we already set this to something incompatible!
				return nil, insertid, fmt.Errorf("incompatible string field with type %v", fields[index].Type)
			}

			// update value
			decoded := make([]byte, base64.StdEncoding.DecodedLen(len(val)))
			numDecoded, err := base64.StdEncoding.Decode(decoded, val)
			if err != nil {
				return nil, insertid, err
			}
			result = append(result, sqltypes.MakeString(decoded[:numDecoded]))
		default:
			return nil, insertid, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
		}

//.........这里部分代码省略.........

// parsePkTuple parses something like (null 1 'bmFtZQ==' ). For numbers, the default
// type is Int64. If an unsigned number that can't fit in an int64 is seen, then the
// type is set to Uint64. In such cases, if a negative number was previously seen, the
// function returns an error.
func parsePkTuple(tokenizer *sqlparser.Tokenizer, insertid int64, fields []*querypb.Field, hasNegatives []bool) (*querypb.Row, int64, error) {
	result := &querypb.Row{}

	index := 0
	for typ, val := tokenizer.Scan(); typ != ')'; typ, val = tokenizer.Scan() {
		if index >= len(fields) {
			return nil, insertid, fmt.Errorf("length mismatch in values")
		}

		switch typ {
		case '-':
			hasNegatives[index] = true
			typ2, val2 := tokenizer.Scan()
			if typ2 != sqlparser.NUMBER {
				return nil, insertid, fmt.Errorf("expecting number after '-'")
			}
			fullVal := append([]byte{'-'}, val2...)
			if _, err := strconv.ParseInt(string(fullVal), 0, 64); err != nil {
				return nil, insertid, err
			}
			switch fields[index].Type {
			case sqltypes.Null:
				fields[index].Type = sqltypes.Int64
			case sqltypes.Int64:
				// no-op
			default:
				return nil, insertid, fmt.Errorf("incompatible negative number field with type %v", fields[index].Type)
			}

			result.Lengths = append(result.Lengths, int64(len(fullVal)))
			result.Values = append(result.Values, fullVal...)
		case sqlparser.NUMBER:
			unsigned, err := strconv.ParseUint(string(val), 0, 64)
			if err != nil {
				return nil, insertid, err
			}
			if unsigned > uint64(9223372036854775807) {
				// Number is a uint64 that can't fit in an int64.
				if hasNegatives[index] {
					return nil, insertid, fmt.Errorf("incompatible unsigned number field with type %v", fields[index].Type)
				}
				switch fields[index].Type {
				case sqltypes.Null, sqltypes.Int64:
					fields[index].Type = sqltypes.Uint64
				case sqltypes.Uint64:
					// no-op
				default:
					return nil, insertid, fmt.Errorf("incompatible number field with type %v", fields[index].Type)
				}
			} else {
				// Could be int64 or uint64.
				switch fields[index].Type {
				case sqltypes.Null:
					fields[index].Type = sqltypes.Int64
				case sqltypes.Int64, sqltypes.Uint64:
					// no-op
				default:
					return nil, insertid, fmt.Errorf("incompatible number field with type %v", fields[index].Type)
				}
			}

			result.Lengths = append(result.Lengths, int64(len(val)))
			result.Values = append(result.Values, val...)
		case sqlparser.NULL:
			switch fields[index].Type {
			case sqltypes.Null:
				fields[index].Type = sqltypes.Int64
			case sqltypes.Int64, sqltypes.Uint64:
				// no-op
			default:
				return nil, insertid, fmt.Errorf("incompatible auto-increment field with type %v", fields[index].Type)
			}

			v := strconv.AppendInt(nil, insertid, 10)
			result.Lengths = append(result.Lengths, int64(len(v)))
			result.Values = append(result.Values, v...)
			insertid++
		case sqlparser.STRING:
			switch fields[index].Type {
			case sqltypes.Null:
				fields[index].Type = sqltypes.VarBinary
			case sqltypes.VarBinary:
				// no-op
			default:
				return nil, insertid, fmt.Errorf("incompatible string field with type %v", fields[index].Type)
			}

			decoded := make([]byte, base64.StdEncoding.DecodedLen(len(val)))
			numDecoded, err := base64.StdEncoding.Decode(decoded, val)
			if err != nil {
				return nil, insertid, err
			}
			result.Lengths = append(result.Lengths, int64(numDecoded))
			result.Values = append(result.Values, decoded[:numDecoded]...)
		default:
			return nil, insertid, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
//.........这里部分代码省略.........