Java StreamTokenizer.slashStarComments方法代码示例

import java.io.StreamTokenizer; //导入方法依赖的package包/类
public JaasConfig(String loginContextName, String jaasConfigParams) {
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(jaasConfigParams));
    tokenizer.slashSlashComments(true);
    tokenizer.slashStarComments(true);
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('$', '$');

    try {
        configEntries = new ArrayList<>();
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            configEntries.add(parseAppConfigurationEntry(tokenizer));
        }
        if (configEntries.isEmpty())
            throw new IllegalArgumentException("Login module not specified in JAAS config");

        this.loginContextName = loginContextName;

    } catch (IOException e) {
        throw new KafkaException("Unexpected exception while parsing JAAS config");
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private Void parse(Reader reader) throws ParseException, IOException
{
    StreamTokenizer st = new StreamTokenizer(reader);
    st.eolIsSignificant(true);
    st.wordChars((int) '_', (int) '_');
    st.parseNumbers();
    st.quoteChar((int) '"');
    // These calls caused comments to be discarded
    st.slashSlashComments(true);
    st.slashStarComments(true);

    // Parse the file
    ParserState currentState = this.getBeginningOfLineState();
    while (currentState != null)
    {
        currentState = currentState.parse(st);
    }
    return null;
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private void readConfig(Reader reader, HashMap newConfig)
		throws IOException {

	if (!(reader instanceof BufferedReader))
		reader = new BufferedReader(reader);

	st = new StreamTokenizer(reader);
	st.quoteChar('"');
	st.wordChars('$', '$');
	st.wordChars('_', '_');
	st.wordChars('-', '-');
	st.lowerCaseMode(false);
	st.slashSlashComments(true);
	st.slashStarComments(true);
	st.eolIsSignificant(true);

	lookahead = nextToken();
	while (lookahead != StreamTokenizer.TT_EOF) {
		parseLoginEntry(newConfig);
	}
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
/**
 * This method sets the syntax of the StreamTokenizer. i.e. set the
 * whitespace, comment and delimit chars.
 * 
 */
protected void setSyntax(StreamTokenizer tk) {
  tk.resetSyntax();
  tk.eolIsSignificant(false);
  tk.slashStarComments(true);
  tk.slashSlashComments(true);
  tk.whitespaceChars(0, ' ');
  tk.wordChars(' ' + 1, '\u00ff');
  tk.ordinaryChar('[');
  tk.ordinaryChar(']');
  tk.ordinaryChar('{');
  tk.ordinaryChar('}');
  tk.ordinaryChar('-');
  tk.ordinaryChar('>');
  tk.ordinaryChar('/');
  tk.ordinaryChar('*');
  tk.quoteChar('"');
  tk.whitespaceChars(';', ';');
  tk.ordinaryChar('=');
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
public static String normalize(String sql, boolean removeComments) {
	if (removeComments)
		sql = sql.replace("--", "//");
	StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(sql));
	tokenizer.resetSyntax();
	tokenizer.wordChars('A', 'Z');
	tokenizer.wordChars('a', 'z');
	tokenizer.wordChars('0', '9');
	tokenizer.wordChars('_', '_');
	tokenizer.whitespaceChars(' ', ' ');
	tokenizer.whitespaceChars('\n', '\n');
	tokenizer.whitespaceChars('\r', '\r');
	tokenizer.whitespaceChars('\t', '\t');
	tokenizer.quoteChar('\'');
	tokenizer.quoteChar('"');
	if (removeComments) {
		tokenizer.slashStarComments(true);
		tokenizer.slashSlashComments(true);
	}
	StringBuilder builder = new StringBuilder();
	int lastTtype = StreamTokenizer.TT_EOF;
	try {
		while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
			int ttype = tokenizer.ttype;
			if (builder.length() > 0 // insert space if this is not the beginning of the string 
					&& ttype != ')' && ttype != ',' && lastTtype != '(' // no space for brackets and lists
					&& lastTtype != '.' && ttype != '.' // no space around '.' 
					&& !(lastTtype == '/' && ttype =='*') // preserve /* if it has not been filtered out 
					&& !(lastTtype == '-' && ttype =='-') // preserve -- if it has not been filtered out
					&& !(lastTtype == '*' && ttype =='/')) // preserve */ if it has not been filtered out
				builder.append(' ');
			switch (ttype) {
				case StreamTokenizer.TT_WORD: builder.append(tokenizer.sval); break;
				case StreamTokenizer.TT_NUMBER: builder.append(renderNumber(tokenizer)); break;
				case '"': builder.append('"').append(tokenizer.sval).append('"'); break;
				case '\'': builder.append('\'').append(tokenizer.sval).append('\''); break;
				default: builder.append((char) ttype);
			}				
			lastTtype = ttype;
		}
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
	return builder.toString();
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private static void configureTokenizer(StreamTokenizer tokenizer) {
    tokenizer.wordChars('_', '_');
    tokenizer.slashSlashComments(true);
    tokenizer.slashStarComments(true);
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
/**
 * 解析 sql 文本，把所有 sql 作为结果返回。
 */
public static String[] splitSql(String sqlText, boolean slashComments) throws IOException {
    if (!sqlText.endsWith(";")) {
        sqlText += ";";
    }

    Reader r = new StringReader(sqlText);
    StreamTokenizer token = new StreamTokenizer(r);

    token.resetSyntax();// 重置此标记生成器的语法表，使所有字符都成为“普通”字符。
    token.slashStarComments(slashComments);// 确定标记生成器是否识别 C 样式注释。
    token.quoteChar('\'');// 指定此字符的匹配对分隔此标记生成器中的字符串常量

    List<String> sqlList = new ArrayList<String>();
    StringBuffer sb = new StringBuffer();

    int flag = StreamTokenizer.TT_EOF;
    while (StreamTokenizer.TT_EOF != (flag = token.nextToken())) {
        switch (flag) {
        case StreamTokenizer.TT_NUMBER:
            sb.append(token.nval);
            break;
        case StreamTokenizer.TT_WORD:
            sb.append(token.sval);
            break;
        case '\'':
            sb.append('\'').append(token.sval).append('\'');
            break;
        case ';':
            if ((!sb.toString().trim().startsWith("--") || !slashComments)
                    && !StringUtils.isBlank(sb.toString().trim())) {
                sqlList.add(sb.toString());
            }
            sb = new StringBuffer();
            break;
        default:
            sb.append((char) token.ttype);
            break;
        }
    }
    return sqlList.toArray(new String[sqlList.size()]);
}