Fix (or just change?) how we report comments vs newlines.
Multi-line comments now swallow internal newlines and don't insert a fake one
in the token stream, so this works like GNU cpp now:
#if /*
*/1
This should be included by the preprocessor, believe it or not.
#endif
Single-line comments no longer swallow their terminating endline, which makes
this case work:
#if BLAH // The preprocessor expects a newline token here.
#endif
--- a/mojoshader_lexer.c Fri Feb 15 15:01:50 2013 -0500
+++ b/mojoshader_lexer.c Fri Feb 15 15:10:33 2013 -0500
@@ -65,7 +65,6 @@
const uchar *matchptr;
const uchar *limit = cursor + s->bytes_left;
int eoi = 0;
- int saw_newline = 0;
@@ -1139,8 +1138,6 @@
yy186:
{
s->line++;
- token = matchptr;
- saw_newline = 1;
goto multilinecomment;
}
yy187:
@@ -1165,8 +1162,6 @@
{
if (s->report_comments)
RET(TOKEN_MULTI_COMMENT);
- else if (saw_newline)
- RET('\n');
else if (s->report_whitespace)
RET(' ');
goto scanner_loop;
@@ -1194,7 +1189,10 @@
{
s->line++;
if (s->report_comments)
+ {
+ cursor = matchptr; // so we RET('\n') next.
RET(TOKEN_SINGLE_COMMENT);
+ }
token = matchptr;
RET('\n');
}
--- a/mojoshader_lexer.re Fri Feb 15 15:01:50 2013 -0500
+++ b/mojoshader_lexer.re Fri Feb 15 15:10:33 2013 -0500
@@ -64,7 +64,6 @@
const uchar *matchptr;
const uchar *limit = cursor + s->bytes_left;
int eoi = 0;
- int saw_newline = 0;
/*!re2c
ANY = [\000-\377];
@@ -171,16 +170,12 @@
"*\/" {
if (s->report_comments)
RET(TOKEN_MULTI_COMMENT);
- else if (saw_newline)
- RET('\n');
else if (s->report_whitespace)
RET(' ');
goto scanner_loop;
}
NEWLINE {
s->line++;
- token = matchptr;
- saw_newline = 1;
goto multilinecomment;
}
"\000" {
@@ -198,7 +193,10 @@
NEWLINE {
s->line++;
if (s->report_comments)
+ {
+ cursor = matchptr; // so we RET('\n') next.
RET(TOKEN_SINGLE_COMMENT);
+ }
token = matchptr;
RET('\n');
}