From 0ca04655634ada83ca97213086f96ef6c71fad29 Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <icculus@icculus.org>
Date: Fri, 15 Feb 2013 15:10:33 -0500
Subject: [PATCH] Fix (or just change?) how we report comments vs newlines.

Multi-line comments now swallow internal newlines and don't insert a fake one
in the token stream, so this works like GNU cpp now:

    #if /*
    */1
    This should be included by the preprocessor, believe it or not.
    #endif

Single-line comments no longer swallow their terminating endline, which makes
this case work:

    #if BLAH  // The preprocessor expects a newline token here.
    #endif
---
 mojoshader_lexer.c  | 8 +++-----
 mojoshader_lexer.re | 8 +++-----
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/mojoshader_lexer.c b/mojoshader_lexer.c
index 89b9847b..01bc62d9 100644
--- a/mojoshader_lexer.c
+++ b/mojoshader_lexer.c
@@ -65,7 +65,6 @@ Token preprocessor_lexer(IncludeState *s)
     const uchar *matchptr;
     const uchar *limit = cursor + s->bytes_left;
     int eoi = 0;
-    int saw_newline = 0;
 
 
 
@@ -1139,8 +1138,6 @@ Token preprocessor_lexer(IncludeState *s)
 yy186:
 	{
                         s->line++;
-                        token = matchptr;
-                        saw_newline = 1;
                         goto multilinecomment;
                     }
 yy187:
@@ -1165,8 +1162,6 @@ Token preprocessor_lexer(IncludeState *s)
 	{
                         if (s->report_comments)
                             RET(TOKEN_MULTI_COMMENT);
-                        else if (saw_newline)
-                            RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
                         goto scanner_loop;
@@ -1194,7 +1189,10 @@ Token preprocessor_lexer(IncludeState *s)
 	{
                         s->line++;
                         if (s->report_comments)
+                        {
+                            cursor = matchptr;  // so we RET('\n') next.
                             RET(TOKEN_SINGLE_COMMENT);
+                        }
                         token = matchptr;
                         RET('\n');
                     }
diff --git a/mojoshader_lexer.re b/mojoshader_lexer.re
index fe2f968c..63e7ca3d 100644
--- a/mojoshader_lexer.re
+++ b/mojoshader_lexer.re
@@ -64,7 +64,6 @@ Token preprocessor_lexer(IncludeState *s)
     const uchar *matchptr;
     const uchar *limit = cursor + s->bytes_left;
     int eoi = 0;
-    int saw_newline = 0;
 
 /*!re2c
     ANY = [\000-\377];
@@ -171,16 +170,12 @@ multilinecomment:
     "*\/"           {
                         if (s->report_comments)
                             RET(TOKEN_MULTI_COMMENT);
-                        else if (saw_newline)
-                            RET('\n');
                         else if (s->report_whitespace)
                             RET(' ');
                         goto scanner_loop;
                     }
     NEWLINE         {
                         s->line++;
-                        token = matchptr;
-                        saw_newline = 1;
                         goto multilinecomment;
                     }
     "\000"          {
@@ -198,7 +193,10 @@ singlelinecomment:
     NEWLINE         {
                         s->line++;
                         if (s->report_comments)
+                        {
+                            cursor = matchptr;  // so we RET('\n') next.
                             RET(TOKEN_SINGLE_COMMENT);
+                        }
                         token = matchptr;
                         RET('\n');
                     }