From bf262fcedfe11a1e929ceacc9af28d1bebebb6cc Mon Sep 17 00:00:00 2001
From: "Ryan C. Gordon" <icculus@icculus.org>
Date: Tue, 10 Feb 2009 17:54:00 -0500
Subject: [PATCH] Added basic lexer debug output.

---
 mojoshader_preprocessor.c | 76 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 74 insertions(+), 2 deletions(-)

diff --git a/mojoshader_preprocessor.c b/mojoshader_preprocessor.c
index 4f46afc3..1a697509 100644
--- a/mojoshader_preprocessor.c
+++ b/mojoshader_preprocessor.c
@@ -10,6 +10,8 @@
 #define __MOJOSHADER_INTERNAL__ 1
 #include "mojoshader_internal.h"
 
+#define DEBUG_TOKENIZER 1
+
 typedef struct DefineHash
 {
     MOJOSHADER_preprocessorDefine define;
@@ -327,8 +329,8 @@ int preprocessor_outofmemory(Preprocessor *_ctx)
 } // preprocessor_outofmemory
 
 
-const char *preprocessor_nexttoken(Preprocessor *_ctx, unsigned int *_len,
-                                    Token *_token)
+static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
+                                             unsigned int *_len, Token *_token)
 {
     Context *ctx = (Context *) _ctx;
 
@@ -389,6 +391,76 @@ const char *preprocessor_nexttoken(Preprocessor *_ctx, unsigned int *_len,
         *_len = (unsigned int) (state->source - state->token);
         return state->token;
     } // while
+
+    assert(0 && "shouldn't hit this code");
+    *_token = TOKEN_UNKNOWN;
+    *_len = 0;
+    return NULL;
+} // _preprocessor_nexttoken
+
+
+const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,
+                                   Token *token)
+{
+    const char *retval = _preprocessor_nexttoken(ctx, len, token);
+
+    #if DEBUG_TOKENIZER
+    printf("TOKEN: \"");
+    unsigned int i;
+    for (i = 0; i < *len; i++)
+    {
+        if (retval[i] == '\n')
+            printf("\\n");
+        else
+            printf("%c", retval[i]);
+    } // for
+    printf("\" (");
+    switch (*token)
+    {
+        #define TOKENCASE(x) case x: printf("%s", #x); break
+        TOKENCASE(TOKEN_UNKNOWN);
+        TOKENCASE(TOKEN_IDENTIFIER);
+        TOKENCASE(TOKEN_INT_LITERAL);
+        TOKENCASE(TOKEN_FLOAT_LITERAL);
+        TOKENCASE(TOKEN_STRING_LITERAL);
+        TOKENCASE(TOKEN_ELLIPSIS);
+        TOKENCASE(TOKEN_RSHIFT);
+        TOKENCASE(TOKEN_LSHIFT);
+        TOKENCASE(TOKEN_ANDAND);
+        TOKENCASE(TOKEN_OROR);
+        TOKENCASE(TOKEN_LEQ);
+        TOKENCASE(TOKEN_GEQ);
+        TOKENCASE(TOKEN_EQL);
+        TOKENCASE(TOKEN_NEQ);
+        TOKENCASE(TOKEN_HASHHASH);
+        TOKENCASE(TOKEN_PP_INCLUDE);
+        TOKENCASE(TOKEN_PP_LINE);
+        TOKENCASE(TOKEN_PP_DEFINE);
+        TOKENCASE(TOKEN_PP_UNDEF);
+        TOKENCASE(TOKEN_PP_IF);
+        TOKENCASE(TOKEN_PP_IFDEF);
+        TOKENCASE(TOKEN_PP_IFNDEF);
+        TOKENCASE(TOKEN_PP_ELSE);
+        TOKENCASE(TOKEN_PP_ELIF);
+        TOKENCASE(TOKEN_PP_ENDIF);
+        TOKENCASE(TOKEN_PP_ERROR);
+        TOKENCASE(TOKEN_PP_INCOMPLETE_COMMENT);
+        TOKENCASE(TOKEN_EOI);
+        #undef TOKENCASE
+
+        case ((Token) '\n'):
+            printf("'\\n'");
+            break;
+
+        default:
+            assert(((int)*token) < 256);
+            printf("'%c'", (char) *token);
+            break;
+    } // switch
+    printf(")\n");
+    #endif
+
+    return retval;
 } // preprocessor_nexttoken