Fix (or just change?) how we report comments vs newlines.
authorRyan C. Gordon <icculus@icculus.org>
Fri, 15 Feb 2013 15:10:33 -0500
changeset 1123 3d842e7dd6ad
parent 1122 b333c15bb984
child 1124 464fc9101a7e
Fix (or just change?) how we report comments vs newlines. Multi-line comments now swallow internal newlines and don't insert a fake one in the token stream, so this works like GNU cpp now: #if /* */1 This should be included by the preprocessor, believe it or not. #endif Single-line comments no longer swallow their terminating endline, which makes this case work: #if BLAH // The preprocessor expects a newline token here. #endif
mojoshader_lexer.c
mojoshader_lexer.re
--- a/mojoshader_lexer.c	Fri Feb 15 15:01:50 2013 -0500
+++ b/mojoshader_lexer.c	Fri Feb 15 15:10:33 2013 -0500
@@ -65,7 +65,6 @@
     const uchar *matchptr;
     const uchar *limit = cursor + s->bytes_left;
     int eoi = 0;
-    int saw_newline = 0;
 
 
 
@@ -1139,8 +1138,6 @@
 yy186:
 	{
                         s->line++;
-                        token = matchptr;
-                        saw_newline = 1;
                         goto multilinecomment;
                     }
 yy187:
@@ -1165,8 +1162,6 @@
 	{
                         if (s->report_comments)
                             RET(TOKEN_MULTI_COMMENT);
-                        else if (saw_newline)
-                            RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
                         goto scanner_loop;
@@ -1194,7 +1189,10 @@
 	{
                         s->line++;
                         if (s->report_comments)
+                        {
+                            cursor = matchptr;  // so we RET('\n') next.
                             RET(TOKEN_SINGLE_COMMENT);
+                        }
                         token = matchptr;
                         RET('\n');
                     }
--- a/mojoshader_lexer.re	Fri Feb 15 15:01:50 2013 -0500
+++ b/mojoshader_lexer.re	Fri Feb 15 15:10:33 2013 -0500
@@ -64,7 +64,6 @@
     const uchar *matchptr;
     const uchar *limit = cursor + s->bytes_left;
     int eoi = 0;
-    int saw_newline = 0;
 
 /*!re2c
     ANY = [\000-\377];
@@ -171,16 +170,12 @@
     "*\/"           {
                         if (s->report_comments)
                             RET(TOKEN_MULTI_COMMENT);
-                        else if (saw_newline)
-                            RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
                         goto scanner_loop;
                     }
     NEWLINE         {
                         s->line++;
-                        token = matchptr;
-                        saw_newline = 1;
                         goto multilinecomment;
                     }
     "\000"          {
@@ -198,7 +193,10 @@
     NEWLINE         {
                         s->line++;
                         if (s->report_comments)
+                        {
+                            cursor = matchptr;  // so we RET('\n') next.
                             RET(TOKEN_SINGLE_COMMENT);
+                        }
                         token = matchptr;
                         RET('\n');
                     }