Bogus chars in the lexer now return a token that signifies this.

icculus · Feb 13, 2009 · 0ace954 · 0ace954
1 parent 6ea97f6
commit 0ace954
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 8 deletions.
diff --git a/mojoshader_internal.h b/mojoshader_internal.h
@@ -326,6 +326,7 @@ typedef enum
     TOKEN_PP_ENDIF,
     TOKEN_PP_ERROR,
     TOKEN_PP_INCOMPLETE_COMMENT,
+    TOKEN_PP_BAD_CHARS,
     TOKEN_EOI
 } Token;
 

diff --git a/mojoshader_lexer.re b/mojoshader_lexer.re
@@ -54,6 +54,7 @@ scanner_loop:
 
 /*!re2c
     ANY = [\000-\377];
+    ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
     O = [0-7];
     D = [0-9];
     L = [a-zA-Z_];
@@ -145,7 +146,7 @@ scanner_loop:
 
     WHITESPACE      { goto scanner_loop; }
     NEWLINE         { s->line++; RET('\n'); }
-    ANY             { printf("bad char\n"); goto scanner_loop; }
+    ANY             { goto bad_chars; }
 */
 
 multilinecomment:
@@ -177,17 +178,13 @@ singlelinecomment:
     ANY             { goto singlelinecomment; }
 */
 
-// !!! FIXME
-/*
 bad_chars:
     if (YYLIMIT == YYCURSOR)
-        RET(TOKEN_BAD_TOKEN);
-*/
+        RET(TOKEN_PP_BAD_CHARS);
 
 /*!re2c
-    NEWLINE         { s->line++; goto scanner_loop; }
-    WHITESPACE      { goto scanner_loop; }
-    any             { goto singlelinecomment; }
+    ANYLEGAL        { cursor--; RET(TOKEN_PP_BAD_CHARS); }
+    ANY             { goto bad_chars; }
 */
 
     assert(0 && "Shouldn't hit this code");

diff --git a/mojoshader_preprocessor.c b/mojoshader_preprocessor.c
@@ -140,6 +140,7 @@ void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
         TOKENCASE(TOKEN_PP_ENDIF);
         TOKENCASE(TOKEN_PP_ERROR);
         TOKENCASE(TOKEN_PP_INCOMPLETE_COMMENT);
+        TOKENCASE(TOKEN_PP_BAD_CHARS);
         TOKENCASE(TOKEN_EOI);
         #undef TOKENCASE