C++ Token::IsBlank方法代码示例

// Return the last fetched token skipping push backed token if it exists
Token* SqlParser::GetLastToken()
{
	Token *token = NULL; 

	if(_push_back_token != NULL)
		token = _push_back_token->prev;
	else
		token = _tokens.GetLastNoCurrent();

	// Return previous non-blank, non-comment token
	while(token && (token->IsBlank() == true || token->type == TOKEN_COMMENT))
		token = token->prev;

	return token;
}

// Skip blanks, tabs and newlines
Token* Token::SkipSpaces(Token *token)
{
	if(token == NULL)
		return NULL;

	Token *cur = token;

	// Skip spaces
	while(cur != NULL)
	{
		if(cur->IsBlank() == false)
			break;

		cur = cur->next;
	}

	return cur;
}

// Get next token from the input
Token* SqlParser::GetNextToken()
{
	// Check push back tokens first
	while(_push_back_token != NULL)
	{
		Token *token = _push_back_token;

		// Multiple tokens can be push back
		_push_back_token = token->next;

		if(token->IsBlank() == true || token->type == TOKEN_COMMENT)
			continue;

		return token;
	}

	Token *token = new Token();
	bool exists = false;

	while(true)
	{
		if(_next_start == NULL || _remain_size <= 0)
		{
			delete token;
			return NULL;
		}

		SkipSpaceTokens();
		
		// Check for a word first as it can start with special symbol such as _ @
		exists = GetWordToken(token);

		if(exists == true)
			break;

		// Check for a single char token
		exists = GetSingleCharToken(token);

		if(exists == true)
			break;
	}
	
	return token;
}

// Set Removed flag for the token
void Token::Remove(Token *token, bool remove_spaces_before)
{
	if(token == NULL)
		return;

	token->flags |= TOKEN_REMOVED;

	Token *prev = token->prev;
	Token *next = token->next;

	if(prev == NULL)
		return;

	if(remove_spaces_before == true)
	{
		// Remove spaces token before the removed word (recursively), but only if there is a blank or ) , ;
		// after removed token (otherwise tokens can be merged)
		if(prev->Compare(' ', L' ') == true || prev->Compare('\t', L'\t') == true)
		{
			if(next != NULL)
			{
				if(next->IsBlank() == true || next->Compare(')', L')') == true || 
					next->Compare(',', L',') == true || next->Compare(';', L';') == true)
					Remove(prev);
			}
			else
			// Next token can be not selected yet
			if(token->next_start != NULL && token->remain_size > 1)
			{
				const char *cur = token->next_start;

				if(*cur == ' ' || *cur == ')' || *cur == ',' || *cur == ';')
					Remove(prev);
			}
		}
	}
	// In case cases better to remove spaces after: "SET var = 1" to "var := 1" when SET is removed 
	else
	{
		if(next != NULL && next->Compare(' ', L' ') == true)
			Remove(next);
	}
}

// Get the previous non-blank, non comment token
Token* SqlParser::GetPrevToken(Token *token)
{
	if(token == NULL)
		return NULL;

	Token *prev = NULL;
	Token *cur = token->prev;

	while(cur != NULL)
	{
		if(cur->IsBlank() == false && cur->type != TOKEN_COMMENT)
		{
			prev = cur;
			break;
		}

		cur = cur->prev;
	}

	return prev;
}

// Generate output
void SqlParser::CreateOutputString(const char **output, int *out_size)
{
	if(output == NULL)
		return;

	Token *token = _tokens.GetFirst();

	int len = 0;

	int not_removed = 0;
	int removed = 0;

	// Calculate the output size in bytes and format output
	while(token != NULL)
	{
		bool r = false;
		bool n = false;

		// If token removed its target length is 0
		len += token->GetTargetLength();

		if(token->IsRemoved() == false)
		{
			r = token->Compare('\r', L'\r');

			if(r == false)
				n = token->Compare('\n', L'\n');
		}

		// Check if we need to remove newline (0D0A \r\n on Windows, 0D on Unix)
		if(r == true || n == true)
		{
			// String was not empty and all tokens removed
			if(not_removed == 0 && removed != 0)
			{
				Token::Remove(token);
				len--;

				// If current is \r and next is \n remove \n 
				if(r == true && token->next != NULL && token->next->Compare('\n', L'\n') == true)
				{
					// Its size will be reduced in the next iteration
					Token::Remove(token->next);
				}

				// Remove all spaces in this empty line
				Token *cur = token->prev;

				while(cur != NULL)
				{
					// Remove until new line
					if(cur->Compare('\n', L'\n') || cur->Compare('\r', L'\r'))
						break;

					if(cur->IsBlank() == true && cur->IsRemoved() == false)
					{
						Token::Remove(cur);
						len--;
					}

					cur = cur->prev;
				}
			}

			not_removed = 0;
			removed = 0;
		}
		// Calculate the number of removed and not removed tokens in line
		else
		{
			if(token->IsBlank() == false)
			{
				if(token->IsRemoved() == true)
					removed++;
				else
					not_removed++;
			}
		}

		token = _tokens.GetNext();
	}
	
	if(len == 0)
	{
		*output = NULL;

		if(out_size != NULL)
			*out_size = 0;

		return;
	}

	// Allocate buffer
	char *out = new char[len + 1]; *out = 0;

	token = _tokens.GetFirst();
	int cur_len = 0;

	while(token != NULL)
//.........这里部分代码省略.........