Fix tokenizer to allow '_' in tokens.

This commit is contained in:
Jeff Young 2020-07-23 11:34:45 +01:00
parent c254b331f3
commit 84085df82a
1 changed files with 3 additions and 3 deletions

View File

@ -433,13 +433,13 @@ bool COMPILER::lexDefault( COMPILER::T_TOKEN& aToken )
m_tokenizer.NextChar(); m_tokenizer.NextChar();
return false; return false;
} }
else if( isalpha( ch ) ) else if( isalpha( ch ) || ch == '_' )
{ {
//printf("ALPHA\n"); //printf("ALPHA\n");
current = m_tokenizer.GetChars( []( int c ) -> bool { return isalnum( c ); } ); current = m_tokenizer.GetChars( []( int c ) -> bool { return isalnum( c ) || c == '_'; } );
//printf("Len: %d\n", current.length() ); //printf("Len: %d\n", current.length() );
//printf("id '%s'\n", (const char *) current.c_str() ); //printf("id '%s'\n", (const char *) current.c_str() );
fflush( stdout ); //fflush( stdout );
retval.token = G_IDENTIFIER; retval.token = G_IDENTIFIER;
snprintf( retval.value.value.str, LIBEVAL_MAX_LITERAL_LENGTH, "%s", current.c_str() ); snprintf( retval.value.value.str, LIBEVAL_MAX_LITERAL_LENGTH, "%s", current.c_str() );
m_tokenizer.NextChar( current.length() ); m_tokenizer.NextChar( current.length() );