Allow preprocessor to pass through comments, like GNU cpp does.
authorRyan C. Gordon <icculus@icculus.org>
Thu, 14 Feb 2013 00:46:12 -0500
changeset 1121 4142681f9fda
parent 1120 f655ea9c8ecd
child 1122 b333c15bb984
Allow preprocessor to pass through comments, like GNU cpp does.
mojoshader_internal.h
mojoshader_lexer.c
mojoshader_lexer.re
mojoshader_preprocessor.c
--- a/mojoshader_internal.h	Thu Feb 14 00:15:14 2013 -0500
+++ b/mojoshader_internal.h	Thu Feb 14 00:46:12 2013 -0500
@@ -474,6 +474,13 @@
     TOKEN_HASH,
     TOKEN_HASHHASH,
 
+    // This is returned if the preprocessor isn't stripping comments. Note
+    //  that in asm files, the ';' counts as a single-line comment, same as
+    //  "//". Note that both eat newline tokens: all of the ones inside a
+    //  multiline comment, and the ending newline on a single-line comment.
+    TOKEN_MULTI_COMMENT,
+    TOKEN_SINGLE_COMMENT,
+
     // This is returned at the end of input...no more to process.
     TOKEN_EOI,
 
@@ -543,6 +550,7 @@
     int pushedback;
     const unsigned char *lexer_marker;
     int report_whitespace;
+    int report_comments;
     int asm_comments;
     unsigned int orig_length;
     unsigned int bytes_left;
--- a/mojoshader_lexer.c	Thu Feb 14 00:15:14 2013 -0500
+++ b/mojoshader_lexer.c	Thu Feb 14 00:46:12 2013 -0500
@@ -1163,7 +1163,9 @@
 yy192:
 	++YYCURSOR;
 	{
-                        if (saw_newline)
+                        if (s->report_comments)
+                            RET(TOKEN_MULTI_COMMENT);
+                        else if (saw_newline)
                             RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
@@ -1189,14 +1191,29 @@
 	}
 	++YYCURSOR;
 yy197:
-	{ s->line++; token = matchptr; RET('\n'); }
+	{
+                        s->line++;
+                        if (s->report_comments)
+                            RET(TOKEN_SINGLE_COMMENT);
+                        token = matchptr;
+                        RET('\n');
+                    }
 yy198:
 	yych = *++YYCURSOR;
 	if (yych == '\n') goto yy203;
 	goto yy197;
 yy199:
 	++YYCURSOR;
-	{ if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
+	{
+                        if (eoi)
+                        {
+                            if (s->report_comments)
+                                RET(TOKEN_SINGLE_COMMENT);
+                            else
+                                RET(TOKEN_EOI);
+                        }
+                        goto singlelinecomment;
+                    }
 yy201:
 	++YYCURSOR;
 	{ goto singlelinecomment; }
--- a/mojoshader_lexer.re	Thu Feb 14 00:15:14 2013 -0500
+++ b/mojoshader_lexer.re	Thu Feb 14 00:46:12 2013 -0500
@@ -169,7 +169,9 @@
 // The "*\/" is just to avoid screwing up text editor syntax highlighting.
 /*!re2c
     "*\/"           {
-                        if (saw_newline)
+                        if (s->report_comments)
+                            RET(TOKEN_MULTI_COMMENT);
+                        else if (saw_newline)
                             RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
@@ -193,8 +195,23 @@
     if (YYLIMIT == YYCURSOR) YYFILL(1);
     matchptr = cursor;
 /*!re2c
-    NEWLINE         { s->line++; token = matchptr; RET('\n'); }
-    "\000"          { if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
+    NEWLINE         {
+                        s->line++;
+                        if (s->report_comments)
+                            RET(TOKEN_SINGLE_COMMENT);
+                        token = matchptr;
+                        RET('\n');
+                    }
+    "\000"          {
+                        if (eoi)
+                        {
+                            if (s->report_comments)
+                                RET(TOKEN_SINGLE_COMMENT);
+                            else
+                                RET(TOKEN_EOI);
+                        }
+                        goto singlelinecomment;
+                    }
     ANY             { goto singlelinecomment; }
 */
 
--- a/mojoshader_preprocessor.c	Thu Feb 14 00:15:14 2013 -0500
+++ b/mojoshader_preprocessor.c	Thu Feb 14 00:46:12 2013 -0500
@@ -180,6 +180,8 @@
         TOKENCASE(TOKEN_PP_PRAGMA);
         TOKENCASE(TOKEN_INCOMPLETE_COMMENT);
         TOKENCASE(TOKEN_BAD_CHARS);
+        TOKENCASE(TOKEN_SINGLE_COMMENT);
+        TOKENCASE(TOKEN_MULTI_COMMENT);
         TOKENCASE(TOKEN_EOI);
         TOKENCASE(TOKEN_PREPROCESSING_ERROR);
         #undef TOKENCASE
@@ -544,6 +546,10 @@
         } // if
     } // if
 
+    #if !MATCH_MICROSOFT_PREPROCESSOR
+    state->report_comments = 1;
+    #endif
+
     state->close_callback = close_callback;
     state->source_base = source;
     state->source = source;
@@ -2121,6 +2127,12 @@
                 continue;  // pushed the include_stack.
         } // else if
 
+        // you don't ever see these unless you enable state->report_comments.
+        else if ((token == TOKEN_SINGLE_COMMENT) || (token == TOKEN_MULTI_COMMENT))
+        {
+            print_debug_lexing_position(state);
+        } // else if
+
         else if (token == ((Token) '\n'))
         {
             print_debug_lexing_position(state);