From 0ace9544c91764a8bb0686e3ef55b01073e759b7 Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <icculus@icculus.org>
Date: Thu, 12 Feb 2009 22:57:21 -0500
Subject: [PATCH] Bogus chars in the lexer now return a token that signifies
 this.

---
 mojoshader_internal.h     |  1 +
 mojoshader_lexer.re       | 13 +++++--------
 mojoshader_preprocessor.c |  1 +
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/mojoshader_internal.h b/mojoshader_internal.h
index d944ea5d..af587f66 100644
--- a/mojoshader_internal.h
+++ b/mojoshader_internal.h
@@ -326,6 +326,7 @@ typedef enum
     TOKEN_PP_ENDIF,
     TOKEN_PP_ERROR,
     TOKEN_PP_INCOMPLETE_COMMENT,
+    TOKEN_PP_BAD_CHARS,
     TOKEN_EOI
 } Token;
 
diff --git a/mojoshader_lexer.re b/mojoshader_lexer.re
index 479b0ba3..6934e20e 100644
--- a/mojoshader_lexer.re
+++ b/mojoshader_lexer.re
@@ -54,6 +54,7 @@ scanner_loop:
 
 /*!re2c
     ANY = [\000-\377];
+    ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
     O = [0-7];
     D = [0-9];
     L = [a-zA-Z_];
@@ -145,7 +146,7 @@ scanner_loop:
 
     WHITESPACE      { goto scanner_loop; }
     NEWLINE         { s->line++; RET('\n'); }
-    ANY             { printf("bad char\n"); goto scanner_loop; }
+    ANY             { goto bad_chars; }
 */
 
 multilinecomment:
@@ -177,17 +178,13 @@ singlelinecomment:
     ANY             { goto singlelinecomment; }
 */
 
-// !!! FIXME
-/*
 bad_chars:
     if (YYLIMIT == YYCURSOR)
-        RET(TOKEN_BAD_TOKEN);
-*/
+        RET(TOKEN_PP_BAD_CHARS);
 
 /*!re2c
-    NEWLINE         { s->line++; goto scanner_loop; }
-    WHITESPACE      { goto scanner_loop; }
-    any             { goto singlelinecomment; }
+    ANYLEGAL        { cursor--; RET(TOKEN_PP_BAD_CHARS); }
+    ANY             { goto bad_chars; }
 */
 
     assert(0 && "Shouldn't hit this code");
diff --git a/mojoshader_preprocessor.c b/mojoshader_preprocessor.c
index 121ecd35..ff3aacb1 100644
--- a/mojoshader_preprocessor.c
+++ b/mojoshader_preprocessor.c
@@ -140,6 +140,7 @@ void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
         TOKENCASE(TOKEN_PP_ENDIF);
         TOKENCASE(TOKEN_PP_ERROR);
         TOKENCASE(TOKEN_PP_INCOMPLETE_COMMENT);
+        TOKENCASE(TOKEN_PP_BAD_CHARS);
         TOKENCASE(TOKEN_EOI);
         #undef TOKENCASE