Allow preprocessor to pass through comments, like GNU cpp does.

icculus · Feb 14, 2013 · 00f25f8 · 00f25f8
1 parent 72e69e2
commit 00f25f8
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 6 deletions.
diff --git a/mojoshader_internal.h b/mojoshader_internal.h
@@ -474,6 +474,13 @@ typedef enum
     TOKEN_HASH,
     TOKEN_HASHHASH,
 
+    // This is returned if the preprocessor isn't stripping comments. Note
+    //  that in asm files, the ';' counts as a single-line comment, same as
+    //  "//". Note that both eat newline tokens: all of the ones inside a
+    //  multiline comment, and the ending newline on a single-line comment.
+    TOKEN_MULTI_COMMENT,
+    TOKEN_SINGLE_COMMENT,
+
     // This is returned at the end of input...no more to process.
     TOKEN_EOI,
 
@@ -543,6 +550,7 @@ typedef struct IncludeState
     int pushedback;
     const unsigned char *lexer_marker;
     int report_whitespace;
+    int report_comments;
     int asm_comments;
     unsigned int orig_length;
     unsigned int bytes_left;

diff --git a/mojoshader_lexer.c b/mojoshader_lexer.c
@@ -1163,7 +1163,9 @@ Token preprocessor_lexer(IncludeState *s)
 yy192:
 	++YYCURSOR;
 	{
-                        if (saw_newline)
+                        if (s->report_comments)
+                            RET(TOKEN_MULTI_COMMENT);
+                        else if (saw_newline)
                             RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
@@ -1189,14 +1191,29 @@ Token preprocessor_lexer(IncludeState *s)
 	}
 	++YYCURSOR;
 yy197:
-	{ s->line++; token = matchptr; RET('\n'); }
+	{
+                        s->line++;
+                        if (s->report_comments)
+                            RET(TOKEN_SINGLE_COMMENT);
+                        token = matchptr;
+                        RET('\n');
+                    }
 yy198:
 	yych = *++YYCURSOR;
 	if (yych == '\n') goto yy203;
 	goto yy197;
 yy199:
 	++YYCURSOR;
-	{ if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
+	{
+                        if (eoi)
+                        {
+                            if (s->report_comments)
+                                RET(TOKEN_SINGLE_COMMENT);
+                            else
+                                RET(TOKEN_EOI);
+                        }
+                        goto singlelinecomment;
+                    }
 yy201:
 	++YYCURSOR;
 	{ goto singlelinecomment; }

diff --git a/mojoshader_lexer.re b/mojoshader_lexer.re
@@ -169,7 +169,9 @@ multilinecomment:
 // The "*\/" is just to avoid screwing up text editor syntax highlighting.
 /*!re2c
     "*\/"           {
-                        if (saw_newline)
+                        if (s->report_comments)
+                            RET(TOKEN_MULTI_COMMENT);
+                        else if (saw_newline)
                             RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
@@ -193,8 +195,23 @@ singlelinecomment:
     if (YYLIMIT == YYCURSOR) YYFILL(1);
     matchptr = cursor;
 /*!re2c
-    NEWLINE         { s->line++; token = matchptr; RET('\n'); }
-    "\000"          { if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
+    NEWLINE         {
+                        s->line++;
+                        if (s->report_comments)
+                            RET(TOKEN_SINGLE_COMMENT);
+                        token = matchptr;
+                        RET('\n');
+                    }
+    "\000"          {
+                        if (eoi)
+                        {
+                            if (s->report_comments)
+                                RET(TOKEN_SINGLE_COMMENT);
+                            else
+                                RET(TOKEN_EOI);
+                        }
+                        goto singlelinecomment;
+                    }
     ANY             { goto singlelinecomment; }
 */
 

diff --git a/mojoshader_preprocessor.c b/mojoshader_preprocessor.c
@@ -180,6 +180,8 @@ void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
         TOKENCASE(TOKEN_PP_PRAGMA);
         TOKENCASE(TOKEN_INCOMPLETE_COMMENT);
         TOKENCASE(TOKEN_BAD_CHARS);
+        TOKENCASE(TOKEN_SINGLE_COMMENT);
+        TOKENCASE(TOKEN_MULTI_COMMENT);
         TOKENCASE(TOKEN_EOI);
         TOKENCASE(TOKEN_PREPROCESSING_ERROR);
         #undef TOKENCASE
@@ -544,6 +546,10 @@ static int push_source(Context *ctx, const char *fname, const char *source,
         } // if
     } // if
 
+    #if !MATCH_MICROSOFT_PREPROCESSOR
+    state->report_comments = 1;
+    #endif
+
     state->close_callback = close_callback;
     state->source_base = source;
     state->source = source;
@@ -2121,6 +2127,12 @@ static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
                 continue;  // pushed the include_stack.
         } // else if
 
+        // you don't ever see these unless you enable state->report_comments.
+        else if ((token == TOKEN_SINGLE_COMMENT) || (token == TOKEN_MULTI_COMMENT))
+        {
+            print_debug_lexing_position(state);
+        } // else if
+
         else if (token == ((Token) '\n'))
         {
             print_debug_lexing_position(state);