Fixed lexer to only accept preprocessor directives at start of a line.

icculus · Feb 22, 2009 · ae4cc7c · ae4cc7c
1 parent 8e0abb7
commit ae4cc7c
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 17 deletions.
diff --git a/mojoshader_lexer.re b/mojoshader_lexer.re
@@ -60,16 +60,12 @@ static Token update_state(IncludeState *s, int eoi, const uchar *cur,
 Token preprocessor_lexer(IncludeState *s)
 {
     const uchar *cursor = (const uchar *) s->source;
-    const uchar *token;
+    const uchar *token = cursor;
     const uchar *matchptr;
     const uchar *limit = cursor + s->bytes_left;
     int eoi = 0;
     int saw_newline = 0;
 
-scanner_loop:
-    if (YYLIMIT == YYCURSOR) YYFILL(1);
-    token = cursor;
-
 /*!re2c
     ANY = [\000-\377];
     ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
@@ -86,6 +82,14 @@ scanner_loop:
     WHITESPACE = [ \t\v\f]+;
 */
 
+    // preprocessor directives are only valid at start of line.
+    if (s->tokenval == ((Token) '\n'))
+        goto ppdirective;  // may jump back to scanner_loop.
+
+scanner_loop:
+    if (YYLIMIT == YYCURSOR) YYFILL(1);
+    token = cursor;
+
 /*!re2c
     "\\" [ \t\v\f]* NEWLINE  { s->line++; goto scanner_loop; }
 
@@ -152,18 +156,6 @@ scanner_loop:
 
     "\000"          { if (eoi) { RET(TOKEN_EOI); } goto bad_chars; }
 
-    PP "include"    { RET(TOKEN_PP_INCLUDE); }
-    PP "line"       { RET(TOKEN_PP_LINE); }
-    PP "define"     { RET(TOKEN_PP_DEFINE); }
-    PP "undef"      { RET(TOKEN_PP_UNDEF); }
-    PP "if"         { RET(TOKEN_PP_IF); }
-    PP "ifdef"      { RET(TOKEN_PP_IFDEF); }
-    PP "ifndef"     { RET(TOKEN_PP_IFNDEF); }
-    PP "else"       { RET(TOKEN_PP_ELSE); }
-    PP "elif"       { RET(TOKEN_PP_ELIF); }
-    PP "endif"      { RET(TOKEN_PP_ENDIF); }
-    PP "error"      { RET(TOKEN_PP_ERROR); }
-
     WHITESPACE      { if (s->report_whitespace) RET(' '); goto scanner_loop; }
     NEWLINE         { s->line++; RET('\n'); }
     ANY             { goto bad_chars; }
@@ -204,6 +196,24 @@ singlelinecomment:
     ANY             { goto singlelinecomment; }
 */
 
+ppdirective:
+    if (YYLIMIT == YYCURSOR) YYFILL(1);
+/*!re2c
+        PP "include"    { RET(TOKEN_PP_INCLUDE); }
+        PP "line"       { RET(TOKEN_PP_LINE); }
+        PP "define"     { RET(TOKEN_PP_DEFINE); }
+        PP "undef"      { RET(TOKEN_PP_UNDEF); }
+        PP "if"         { RET(TOKEN_PP_IF); }
+        PP "ifdef"      { RET(TOKEN_PP_IFDEF); }
+        PP "ifndef"     { RET(TOKEN_PP_IFNDEF); }
+        PP "else"       { RET(TOKEN_PP_ELSE); }
+        PP "elif"       { RET(TOKEN_PP_ELIF); }
+        PP "endif"      { RET(TOKEN_PP_ENDIF); }
+        PP "error"      { RET(TOKEN_PP_ERROR); }
+        WHITESPACE      { goto ppdirective; }
+        ANY             { cursor=(const uchar*)s->source; goto scanner_loop; }
+*/
+
 bad_chars:
     if (YYLIMIT == YYCURSOR) YYFILL(1);
 /*!re2c

diff --git a/mojoshader_preprocessor.c b/mojoshader_preprocessor.c
@@ -545,6 +545,7 @@ static int push_source(Context *ctx, const char *fname, const char *source,
     state->source_base = source;
     state->source = source;
     state->token = source;
+    state->tokenval = ((Token) '\n');
     state->orig_length = srclen;
     state->bytes_left = srclen;
     state->line = linenum;