本文整理汇总了Java中org.python.pydev.parser.jython.Token类的典型用法代码示例。如果您正苦于以下问题：Java Token类的具体用法？Java Token怎么用？Java Token使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Token类属于org.python.pydev.parser.jython包，在下文中一共展示了Token类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: fullTokenListWithPos
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
final FastCharStream stream = new FastCharStream(code);
final ITokenManager mng = getPythonTokenizer(stream);
final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
Token nextToken = mng.getNextToken();
while (nextToken.kind != 0) {
if (shouldAdd(nextToken)) {
// TODO: Bad Heurisitc...
tokens.put(
nextToken.getBeginLine() * 500
+ nextToken.getBeginCol(),
new FullToken(nextToken.image, Integer
.toString(nextToken.kind)));
}
nextToken = mng.getNextToken();
}
return tokens;
}
示例2: getTokenListFromCode
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
final FastCharStream stream = new FastCharStream(code);
final ITokenManager mng = getPythonTokenizer(stream);
final List<FullToken> tokens = Lists.newArrayList();
Token nextToken = mng.getNextToken();
while (nextToken.kind != 0) {
if (shouldAdd(nextToken)) {
tokens.add(new FullToken(nextToken.image, Integer
.toString(nextToken.kind)));
}
nextToken = mng.getNextToken();
}
return tokens;
}
示例3: tokenListFromCode
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
@Override
public List<String> tokenListFromCode(final char[] code) {
final FastCharStream stream = new FastCharStream(code);
final ITokenManager mng = getPythonTokenizer(stream);
final List<String> tokens = Lists.newArrayList();
Token nextToken = mng.getNextToken();
while (nextToken.kind != 0) {
if (shouldAdd(nextToken)) {
tokens.add(nextToken.image);
}
nextToken = mng.getNextToken();
}
return tokens;
}
示例4: tokenListWithPos
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
final FastCharStream stream = new FastCharStream(code);
final ITokenManager mng = getPythonTokenizer(stream);
final SortedMap<Integer, String> tokens = Maps.newTreeMap();
Token nextToken = mng.getNextToken();
while (nextToken.kind != 0) {
if (shouldAdd(nextToken)) {
// TODO: Bad Heurisitc...
tokens.put(
nextToken.getBeginLine() * 500
+ nextToken.getBeginCol(), nextToken.image);
}
nextToken = mng.getNextToken();
}
return tokens;
}
示例5: addCustom
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
/**
* Creates a custom token for the given token representation (if possible)
*
* @see #CUSTOM_NOT_CREATED
* @see #CUSTOM_CREATED_NOT_PARENS
* @see #CUSTOM_CREATED_WAS_PARENS
*/
public int addCustom(Token curr, String token) {
Integer id = getTokenToId().get(token);
if (id != null) {
createFromAndSetAsNext(curr, id, token);
int ret = CUSTOM_CREATED_NOT_PARENS;
if (id == getRparenId() || id == getRbracketId() || id == getRbraceId()) {
parens--;
ret = CUSTOM_CREATED_WAS_PARENS;
} else if (id == getLparenId()) {
parens++;
ret = CUSTOM_CREATED_WAS_PARENS;
}
return ret;
}
return CUSTOM_NOT_CREATED;
}
示例6: jj_3R_51
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_51() {
Token xsp;
xsp = jj_scanpos;
if (jj_3R_77()) {
jj_scanpos = xsp;
if (jj_3R_78()) {
jj_scanpos = xsp;
if (jj_3R_79()) {
jj_scanpos = xsp;
if (jj_3R_80())
return true;
}
}
}
return false;
}
示例7: jj_3R_47
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_47() {
Token xsp;
xsp = jj_scanpos;
if (jj_3R_72()) {
jj_scanpos = xsp;
if (jj_3R_73()) {
jj_scanpos = xsp;
if (jj_3R_74()) {
jj_scanpos = xsp;
if (jj_3R_75())
return true;
}
}
}
return false;
}
示例8: jj_3R_131
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_131() {
Token xsp;
xsp = jj_scanpos;
if (jj_3R_148()) {
jj_scanpos = xsp;
if (jj_3R_149()) {
jj_scanpos = xsp;
if (jj_3R_150()) {
jj_scanpos = xsp;
if (jj_3R_151()) {
jj_scanpos = xsp;
if (jj_3R_152()) return true;
}
}
}
}
return false;
}
示例9: jj_3R_188
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_188() {
Token xsp;
xsp = jj_scanpos;
if (jj_3R_191()) {
jj_scanpos = xsp;
if (jj_3R_192()) {
jj_scanpos = xsp;
if (jj_3R_193()) {
jj_scanpos = xsp;
if (jj_3R_194()) {
jj_scanpos = xsp;
if (jj_3R_195()) {
jj_scanpos = xsp;
if (jj_3R_196()) return true;
}
}
}
}
}
return false;
}
示例10: jj_consume_token
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
/** Reinitialise. */
//Removed Reinit}
private Token jj_consume_token(int kind) throws ParseException {
Token oldToken;
if ((oldToken = token).next != null) token = token.next;
else token = token.next = token_source.getNextToken();
jj_ntk = -1;
if (token.kind == kind) {
jj_gen++;
if (++jj_gc > 100) {
jj_gc = 0;
for (int i = 0; i < jj_2_rtns.length; i++) {
JJCalls c = jj_2_rtns[i];
while (c != null) {
if (c.gen < jj_gen) c.first = null;
c = c.next;
}
}
}
return token;
}
token = oldToken;
jj_kind = kind;
throw generateParseException();
}
示例11: jj_scan_token
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_scan_token(int kind) {
if (jj_scanpos == jj_lastpos) {
jj_la--;
if (jj_scanpos.next == null) {
jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
} else {
jj_lastpos = jj_scanpos = jj_scanpos.next;
}
} else {
jj_scanpos = jj_scanpos.next;
}
if (jj_rescan) {
int i = 0; Token tok = token;
while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
if (tok != null) jj_add_error_token(kind, i);
}
if (jj_scanpos.kind != kind) return true;
if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls;
return false;
}
示例12: pass_stmt
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
final public Token pass_stmt() throws ParseException {
/*@bgen(jjtree) pass_stmt */
SimpleNode jjtn000 = builder.openNode(JJTPASS_STMT);
boolean jjtc000 = true;
jjtree.openNodeScope(jjtn000);
jjtreeOpenNodeScope(jjtn000);
Token spStr;
try {
spStr = jj_consume_token(PASS);
jjtree.closeNodeScope(jjtn000, true);
jjtc000 = false;
jjtreeCloseNodeScope(jjtn000);
{
if (true)
return spStr;
}
} finally {
if (jjtc000) {
jjtree.closeNodeScope(jjtn000, true);
jjtreeCloseNodeScope(jjtn000);
}
}
throw new Error("Missing return statement in function");
}
示例13: jj_3R_89
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_89() {
Token xsp;
xsp = jj_scanpos;
if (jj_3R_104()) {
jj_scanpos = xsp;
if (jj_3R_105()) {
jj_scanpos = xsp;
if (jj_3R_106()) {
jj_scanpos = xsp;
if (jj_3R_107()) {
jj_scanpos = xsp;
if (jj_3R_108())
return true;
}
}
}
}
return false;
}
示例14: getTokenFromString
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
@Override
public FullToken getTokenFromString(final String token) {
final FastCharStream stream = new FastCharStream(token.toCharArray());
final ITokenManager mng = getPythonTokenizer(stream);
final Token pyToken = mng.getNextToken();
return new FullToken(pyToken.image, Integer.toString(pyToken.kind));
}
示例15: jj_3R_182
import org.python.pydev.parser.jython.Token; //导入依赖的package包/类
private boolean jj_3R_182() {
if (jj_3R_189()) return true;
Token xsp;
while (true) {
xsp = jj_scanpos;
if (jj_3R_239()) { jj_scanpos = xsp; break; }
}
return false;
}