mojoshader_preprocessor.c
changeset 587 202354e004fc
parent 586 321a19a62989
child 597 832dbfa63509
equal deleted inserted replaced
586:321a19a62989 587:202354e004fc
     8  */
     8  */
     9 
     9 
    10 #define __MOJOSHADER_INTERNAL__ 1
    10 #define __MOJOSHADER_INTERNAL__ 1
    11 #include "mojoshader_internal.h"
    11 #include "mojoshader_internal.h"
    12 
    12 
    13 #define DEBUG_TOKENIZER 0
    13 #if DEBUG_PREPROCESSOR
       
    14     #define print_debug_token(token, len, val) \
       
    15         MOJOSHADER_print_debug_token("PREPROCESSOR", token, len, val)
       
    16 #else
       
    17     #define print_debug_token(token, len, val)
       
    18 #endif
    14 
    19 
    15 typedef struct DefineHash
    20 typedef struct DefineHash
    16 {
    21 {
    17     MOJOSHADER_preprocessorDefine define;
    22     MOJOSHADER_preprocessorDefine define;
    18     struct DefineHash *next;
    23     struct DefineHash *next;
    79 {
    84 {
    80     failf(ctx, "%s", reason);
    85     failf(ctx, "%s", reason);
    81 } // fail
    86 } // fail
    82 
    87 
    83 
    88 
    84 // Preprocessor define hashtable stuff...
    89 #if DEBUG_TOKENIZER
    85 
    90 void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
    86 static unsigned char hash_define(const char *sym)
    91                                   const unsigned int tokenlen,
    87 {
    92                                   const Token tokenval)
    88     unsigned char retval = 0;
    93 {
    89     while (sym)
    94     printf("%s TOKEN: \"", subsystem);
    90         retval += *(sym++);
       
    91     return retval;
       
    92 } // hash_define
       
    93 
       
    94 
       
    95 static int add_define(Context *ctx, const char *sym, const char *val)
       
    96 {
       
    97     char *identifier = NULL;
       
    98     char *definition = NULL;
       
    99     const unsigned char hash = hash_define(sym);
       
   100     DefineHash *bucket = ctx->define_hashtable[hash];
       
   101     while (bucket)
       
   102     {
       
   103         if (strcmp(bucket->define.identifier, sym) == 0)
       
   104         {
       
   105             failf(ctx, "'%s' already defined", sym);
       
   106             return 0;
       
   107         } // if
       
   108         bucket = bucket->next;
       
   109     } // while
       
   110 
       
   111     bucket = (DefineHash *) Malloc(ctx, sizeof (DefineHash));
       
   112     if (bucket == NULL)
       
   113         return 0;
       
   114 
       
   115     identifier = (char *) Malloc(ctx, strlen(sym) + 1);
       
   116     definition = (char *) Malloc(ctx, strlen(val) + 1);
       
   117     if ((identifier == NULL) || (definition == NULL))
       
   118     {
       
   119         Free(ctx, identifier);
       
   120         Free(ctx, definition);
       
   121         Free(ctx, bucket);
       
   122         return 0;
       
   123     } // if
       
   124 
       
   125     strcpy(identifier, sym);
       
   126     strcpy(definition, val);
       
   127     bucket->define.identifier = identifier;
       
   128     bucket->define.definition = definition;
       
   129     bucket->next = ctx->define_hashtable[hash];
       
   130     ctx->define_hashtable[hash] = bucket;
       
   131     return 1;
       
   132 } // add_define
       
   133 
       
   134 
       
   135 static int remove_define(Context *ctx, const char *sym)
       
   136 {
       
   137     const unsigned char hash = hash_define(sym);
       
   138     DefineHash *bucket = ctx->define_hashtable[hash];
       
   139     DefineHash *prev = NULL;
       
   140     while (bucket)
       
   141     {
       
   142         if (strcmp(bucket->define.identifier, sym) == 0)
       
   143         {
       
   144             if (prev == NULL)
       
   145                 ctx->define_hashtable[hash] = bucket->next;
       
   146             else
       
   147                 prev->next = bucket->next;
       
   148             Free(ctx, (void *) bucket->define.identifier);
       
   149             Free(ctx, (void *) bucket->define.definition);
       
   150             Free(ctx, bucket);
       
   151             return 1;
       
   152         } // if
       
   153         prev = bucket;
       
   154         bucket = bucket->next;
       
   155     } // while
       
   156 
       
   157     failf(ctx, "'%s' not defined", sym);
       
   158     return 0;
       
   159 } // remove_define
       
   160 
       
   161 
       
   162 static const char *find_define(Context *ctx, const char *sym)
       
   163 {
       
   164     const unsigned char hash = hash_define(sym);
       
   165     DefineHash *bucket = ctx->define_hashtable[hash];
       
   166     while (bucket)
       
   167     {
       
   168         if (strcmp(bucket->define.identifier, sym) == 0)
       
   169             return bucket->define.definition;
       
   170         bucket = bucket->next;
       
   171     } // while
       
   172     return NULL;
       
   173 } // find_define
       
   174 
       
   175 
       
   176 static void free_all_defines(Context *ctx)
       
   177 {
       
   178     int i;
       
   179     for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)
       
   180     {
       
   181         DefineHash *bucket = ctx->define_hashtable[i];
       
   182         ctx->define_hashtable[i] = NULL;
       
   183         while (bucket)
       
   184         {
       
   185             DefineHash *next = bucket->next;
       
   186             Free(ctx, (void *) bucket->define.identifier);
       
   187             Free(ctx, (void *) bucket->define.definition);
       
   188             Free(ctx, bucket);
       
   189             bucket = next;
       
   190         } // while
       
   191     } // for
       
   192 } // find_define
       
   193 
       
   194 
       
   195 static int push_source(Context *ctx, const char *fname, const char *source,
       
   196                        unsigned int srclen, int included)
       
   197 {
       
   198     IncludeState *state = (IncludeState *) Malloc(ctx, sizeof (IncludeState));
       
   199     if (state == NULL)
       
   200         return 0;
       
   201     memset(state, '\0', sizeof (IncludeState));
       
   202 
       
   203     if (fname != NULL)
       
   204     {
       
   205         state->filename = StrDup(ctx, fname);
       
   206         if (state->filename == NULL)
       
   207         {
       
   208             Free(ctx, state);
       
   209             return 0;
       
   210         } // if
       
   211     } // if
       
   212 
       
   213     state->included = included;
       
   214     state->source_base = source;
       
   215     state->source = source;
       
   216     state->token = source;
       
   217     state->bytes_left = srclen;
       
   218     state->line = 1;
       
   219     state->next = ctx->include_stack;
       
   220 
       
   221     ctx->include_stack = state;
       
   222 
       
   223     return 1;
       
   224 } // push_source
       
   225 
       
   226 
       
   227 static void pop_source(Context *ctx)
       
   228 {
       
   229     IncludeState *state = ctx->include_stack;
       
   230     if (state == NULL)
       
   231         return;
       
   232 
       
   233     if (state->included)
       
   234     {
       
   235         ctx->close_callback(state->source_base, ctx->malloc,
       
   236                             ctx->free, ctx->malloc_data);
       
   237     } // if
       
   238 
       
   239     ctx->include_stack = state->next;
       
   240     Free(ctx, state->filename);
       
   241     Free(ctx, state);
       
   242 } // pop_source
       
   243 
       
   244 
       
   245 Preprocessor *preprocessor_start(const char *fname, const char *source,
       
   246                             unsigned int sourcelen,
       
   247                             MOJOSHADER_includeOpen open_callback,
       
   248                             MOJOSHADER_includeClose close_callback,
       
   249                             const MOJOSHADER_preprocessorDefine **defines,
       
   250                             unsigned int define_count,
       
   251                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
       
   252 {
       
   253     int okay = 1;
       
   254     int i = 0;
       
   255 
       
   256     // the preprocessor is internal-only, so we verify all these are != NULL.
       
   257     assert(m != NULL);
       
   258     assert(f != NULL);
       
   259     assert(open_callback != NULL);
       
   260     assert(close_callback != NULL);
       
   261 
       
   262     Context *ctx = (Context *) m(sizeof (Context), d);
       
   263     if (ctx == NULL)
       
   264         return 0;
       
   265 
       
   266     memset(ctx, '\0', sizeof (Context));
       
   267     ctx->malloc = m;
       
   268     ctx->free = f;
       
   269     ctx->malloc_data = d;
       
   270     ctx->open_callback = open_callback;
       
   271     ctx->close_callback = close_callback;
       
   272 
       
   273     for (i = 0; i < define_count; i++)
       
   274     {
       
   275         if (!add_define(ctx, defines[i]->identifier, defines[i]->definition))
       
   276         {
       
   277             okay = 0;
       
   278             break;
       
   279         } // if
       
   280     } // for
       
   281 
       
   282     if ((okay) && (!push_source(ctx, fname, source, sourcelen, 0)))
       
   283         okay = 0;
       
   284 
       
   285     if (!okay)
       
   286     {
       
   287         preprocessor_end((Preprocessor *) ctx);
       
   288         return NULL;
       
   289     } // if
       
   290 
       
   291     return (Preprocessor *) ctx;
       
   292 } // preprocessor_start
       
   293 
       
   294 
       
   295 void preprocessor_end(Preprocessor *_ctx)
       
   296 {
       
   297     Context *ctx = (Context *) _ctx;
       
   298     if (ctx == NULL)
       
   299         return;
       
   300 
       
   301     while (ctx->include_stack != NULL)
       
   302         pop_source(ctx);
       
   303 
       
   304     free_all_defines(ctx);
       
   305 
       
   306     Free(ctx, ctx);
       
   307 } // preprocessor_end
       
   308 
       
   309 
       
   310 const char *preprocessor_error(Preprocessor *_ctx)
       
   311 {
       
   312     Context *ctx = (Context *) _ctx;
       
   313     if (ctx->isfail)
       
   314     {
       
   315         ctx->isfail = 0;
       
   316         return ctx->failstr;
       
   317     } // if
       
   318 
       
   319     return NULL;
       
   320 } // preprocessor_error
       
   321 
       
   322 
       
   323 int preprocessor_outofmemory(Preprocessor *_ctx)
       
   324 {
       
   325     Context *ctx = (Context *) _ctx;
       
   326     return ctx->out_of_memory;
       
   327 } // preprocessor_outofmemory
       
   328 
       
   329 
       
   330 static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
       
   331                                              unsigned int *_len, Token *_token)
       
   332 {
       
   333     Context *ctx = (Context *) _ctx;
       
   334 
       
   335     while (1)
       
   336     {
       
   337         IncludeState *state = ctx->include_stack;
       
   338         if (state == NULL)
       
   339         {
       
   340             *_token = TOKEN_EOI;
       
   341             *_len = 0;
       
   342             return NULL;  // we're done!
       
   343         } // if
       
   344 
       
   345         Token token = preprocessor_internal_lexer(state);
       
   346         if (token == TOKEN_EOI)
       
   347         {
       
   348             assert(state->bytes_left == 0);
       
   349             pop_source(ctx);
       
   350             continue;  // pick up again after parent's #include line.
       
   351         } // if
       
   352 
       
   353         else if (token == TOKEN_PP_INCOMPLETE_COMMENT)
       
   354         {
       
   355             fail(ctx, "Incomplete multiline comment");
       
   356             continue;  // !!! FIXME: we should probably return TOKEN_ERROR or something.
       
   357         } // else if
       
   358 
       
   359         *_token = token;
       
   360         *_len = (unsigned int) (state->source - state->token);
       
   361         return state->token;
       
   362     } // while
       
   363 
       
   364     assert(0 && "shouldn't hit this code");
       
   365     *_token = TOKEN_UNKNOWN;
       
   366     *_len = 0;
       
   367     return NULL;
       
   368 } // _preprocessor_nexttoken
       
   369 
       
   370 
       
   371 const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,
       
   372                                    Token *token)
       
   373 {
       
   374     const char *retval = _preprocessor_nexttoken(ctx, len, token);
       
   375 
       
   376     #if DEBUG_TOKENIZER
       
   377     printf("PREPROCESSOR TOKEN: \"");
       
   378     unsigned int i;
    95     unsigned int i;
   379     for (i = 0; i < *len; i++)
    96     for (i = 0; i < tokenlen; i++)
   380     {
    97     {
   381         if (retval[i] == '\n')
    98         if (token[i] == '\n')
   382             printf("\\n");
    99             printf("\\n");
   383         else
   100         else
   384             printf("%c", retval[i]);
   101             printf("%c", token[i]);
   385     } // for
   102     } // for
   386     printf("\" (");
   103     printf("\" (");
   387     switch (*token)
   104     switch (tokenval)
   388     {
   105     {
   389         #define TOKENCASE(x) case x: printf("%s", #x); break
   106         #define TOKENCASE(x) case x: printf("%s", #x); break
   390         TOKENCASE(TOKEN_UNKNOWN);
   107         TOKENCASE(TOKEN_UNKNOWN);
   391         TOKENCASE(TOKEN_IDENTIFIER);
   108         TOKENCASE(TOKEN_IDENTIFIER);
   392         TOKENCASE(TOKEN_INT_LITERAL);
   109         TOKENCASE(TOKEN_INT_LITERAL);
   393         TOKENCASE(TOKEN_FLOAT_LITERAL);
   110         TOKENCASE(TOKEN_FLOAT_LITERAL);
   394         TOKENCASE(TOKEN_STRING_LITERAL);
   111         TOKENCASE(TOKEN_STRING_LITERAL);
   395         TOKENCASE(TOKEN_RSHIFTASSIGN);
       
   396         TOKENCASE(TOKEN_LSHIFTASSIGN);
       
   397         TOKENCASE(TOKEN_ADDASSIGN);
   112         TOKENCASE(TOKEN_ADDASSIGN);
   398         TOKENCASE(TOKEN_SUBASSIGN);
   113         TOKENCASE(TOKEN_SUBASSIGN);
   399         TOKENCASE(TOKEN_MULTASSIGN);
   114         TOKENCASE(TOKEN_MULTASSIGN);
   400         TOKENCASE(TOKEN_DIVASSIGN);
   115         TOKENCASE(TOKEN_DIVASSIGN);
   401         TOKENCASE(TOKEN_MODASSIGN);
   116         TOKENCASE(TOKEN_MODASSIGN);
   431         case ((Token) '\n'):
   146         case ((Token) '\n'):
   432             printf("'\\n'");
   147             printf("'\\n'");
   433             break;
   148             break;
   434 
   149 
   435         default:
   150         default:
   436             assert(((int)*token) < 256);
   151             assert(((int)tokenval) < 256);
   437             printf("'%c'", (char) *token);
   152             printf("'%c'", (char) tokenval);
   438             break;
   153             break;
   439     } // switch
   154     } // switch
   440     printf(")\n");
   155     printf(")\n");
   441     #endif
   156 } // MOJOSHADER_print_debug_token
   442 
   157 #endif
       
   158 
       
   159 
       
   160 // Preprocessor define hashtable stuff...
       
   161 
       
   162 static unsigned char hash_define(const char *sym)
       
   163 {
       
   164     unsigned char retval = 0;
       
   165     while (sym)
       
   166         retval += *(sym++);
       
   167     return retval;
       
   168 } // hash_define
       
   169 
       
   170 
       
   171 static int add_define(Context *ctx, const char *sym, const char *val)
       
   172 {
       
   173     char *identifier = NULL;
       
   174     char *definition = NULL;
       
   175     const unsigned char hash = hash_define(sym);
       
   176     DefineHash *bucket = ctx->define_hashtable[hash];
       
   177     while (bucket)
       
   178     {
       
   179         if (strcmp(bucket->define.identifier, sym) == 0)
       
   180         {
       
   181             failf(ctx, "'%s' already defined", sym);
       
   182             return 0;
       
   183         } // if
       
   184         bucket = bucket->next;
       
   185     } // while
       
   186 
       
   187     bucket = (DefineHash *) Malloc(ctx, sizeof (DefineHash));
       
   188     if (bucket == NULL)
       
   189         return 0;
       
   190 
       
   191     identifier = (char *) Malloc(ctx, strlen(sym) + 1);
       
   192     definition = (char *) Malloc(ctx, strlen(val) + 1);
       
   193     if ((identifier == NULL) || (definition == NULL))
       
   194     {
       
   195         Free(ctx, identifier);
       
   196         Free(ctx, definition);
       
   197         Free(ctx, bucket);
       
   198         return 0;
       
   199     } // if
       
   200 
       
   201     strcpy(identifier, sym);
       
   202     strcpy(definition, val);
       
   203     bucket->define.identifier = identifier;
       
   204     bucket->define.definition = definition;
       
   205     bucket->next = ctx->define_hashtable[hash];
       
   206     ctx->define_hashtable[hash] = bucket;
       
   207     return 1;
       
   208 } // add_define
       
   209 
       
   210 
       
   211 static int remove_define(Context *ctx, const char *sym)
       
   212 {
       
   213     const unsigned char hash = hash_define(sym);
       
   214     DefineHash *bucket = ctx->define_hashtable[hash];
       
   215     DefineHash *prev = NULL;
       
   216     while (bucket)
       
   217     {
       
   218         if (strcmp(bucket->define.identifier, sym) == 0)
       
   219         {
       
   220             if (prev == NULL)
       
   221                 ctx->define_hashtable[hash] = bucket->next;
       
   222             else
       
   223                 prev->next = bucket->next;
       
   224             Free(ctx, (void *) bucket->define.identifier);
       
   225             Free(ctx, (void *) bucket->define.definition);
       
   226             Free(ctx, bucket);
       
   227             return 1;
       
   228         } // if
       
   229         prev = bucket;
       
   230         bucket = bucket->next;
       
   231     } // while
       
   232 
       
   233     failf(ctx, "'%s' not defined", sym);
       
   234     return 0;
       
   235 } // remove_define
       
   236 
       
   237 
       
   238 static const char *find_define(Context *ctx, const char *sym)
       
   239 {
       
   240     const unsigned char hash = hash_define(sym);
       
   241     DefineHash *bucket = ctx->define_hashtable[hash];
       
   242     while (bucket)
       
   243     {
       
   244         if (strcmp(bucket->define.identifier, sym) == 0)
       
   245             return bucket->define.definition;
       
   246         bucket = bucket->next;
       
   247     } // while
       
   248     return NULL;
       
   249 } // find_define
       
   250 
       
   251 
       
   252 static void free_all_defines(Context *ctx)
       
   253 {
       
   254     int i;
       
   255     for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)
       
   256     {
       
   257         DefineHash *bucket = ctx->define_hashtable[i];
       
   258         ctx->define_hashtable[i] = NULL;
       
   259         while (bucket)
       
   260         {
       
   261             DefineHash *next = bucket->next;
       
   262             Free(ctx, (void *) bucket->define.identifier);
       
   263             Free(ctx, (void *) bucket->define.definition);
       
   264             Free(ctx, bucket);
       
   265             bucket = next;
       
   266         } // while
       
   267     } // for
       
   268 } // find_define
       
   269 
       
   270 
       
   271 static int push_source(Context *ctx, const char *fname, const char *source,
       
   272                        unsigned int srclen, int included)
       
   273 {
       
   274     IncludeState *state = (IncludeState *) Malloc(ctx, sizeof (IncludeState));
       
   275     if (state == NULL)
       
   276         return 0;
       
   277     memset(state, '\0', sizeof (IncludeState));
       
   278 
       
   279     if (fname != NULL)
       
   280     {
       
   281         state->filename = StrDup(ctx, fname);
       
   282         if (state->filename == NULL)
       
   283         {
       
   284             Free(ctx, state);
       
   285             return 0;
       
   286         } // if
       
   287     } // if
       
   288 
       
   289     state->included = included;
       
   290     state->source_base = source;
       
   291     state->source = source;
       
   292     state->token = source;
       
   293     state->bytes_left = srclen;
       
   294     state->line = 1;
       
   295     state->next = ctx->include_stack;
       
   296 
       
   297     ctx->include_stack = state;
       
   298 
       
   299     return 1;
       
   300 } // push_source
       
   301 
       
   302 
       
   303 static void pop_source(Context *ctx)
       
   304 {
       
   305     IncludeState *state = ctx->include_stack;
       
   306     if (state == NULL)
       
   307         return;
       
   308 
       
   309     if (state->included)
       
   310     {
       
   311         ctx->close_callback(state->source_base, ctx->malloc,
       
   312                             ctx->free, ctx->malloc_data);
       
   313     } // if
       
   314 
       
   315     ctx->include_stack = state->next;
       
   316     Free(ctx, state->filename);
       
   317     Free(ctx, state);
       
   318 } // pop_source
       
   319 
       
   320 
       
   321 Preprocessor *preprocessor_start(const char *fname, const char *source,
       
   322                             unsigned int sourcelen,
       
   323                             MOJOSHADER_includeOpen open_callback,
       
   324                             MOJOSHADER_includeClose close_callback,
       
   325                             const MOJOSHADER_preprocessorDefine **defines,
       
   326                             unsigned int define_count,
       
   327                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
       
   328 {
       
   329     int okay = 1;
       
   330     int i = 0;
       
   331 
       
   332     // the preprocessor is internal-only, so we verify all these are != NULL.
       
   333     assert(m != NULL);
       
   334     assert(f != NULL);
       
   335     assert(open_callback != NULL);
       
   336     assert(close_callback != NULL);
       
   337 
       
   338     Context *ctx = (Context *) m(sizeof (Context), d);
       
   339     if (ctx == NULL)
       
   340         return 0;
       
   341 
       
   342     memset(ctx, '\0', sizeof (Context));
       
   343     ctx->malloc = m;
       
   344     ctx->free = f;
       
   345     ctx->malloc_data = d;
       
   346     ctx->open_callback = open_callback;
       
   347     ctx->close_callback = close_callback;
       
   348 
       
   349     for (i = 0; i < define_count; i++)
       
   350     {
       
   351         if (!add_define(ctx, defines[i]->identifier, defines[i]->definition))
       
   352         {
       
   353             okay = 0;
       
   354             break;
       
   355         } // if
       
   356     } // for
       
   357 
       
   358     if ((okay) && (!push_source(ctx, fname, source, sourcelen, 0)))
       
   359         okay = 0;
       
   360 
       
   361     if (!okay)
       
   362     {
       
   363         preprocessor_end((Preprocessor *) ctx);
       
   364         return NULL;
       
   365     } // if
       
   366 
       
   367     return (Preprocessor *) ctx;
       
   368 } // preprocessor_start
       
   369 
       
   370 
       
   371 void preprocessor_end(Preprocessor *_ctx)
       
   372 {
       
   373     Context *ctx = (Context *) _ctx;
       
   374     if (ctx == NULL)
       
   375         return;
       
   376 
       
   377     while (ctx->include_stack != NULL)
       
   378         pop_source(ctx);
       
   379 
       
   380     free_all_defines(ctx);
       
   381 
       
   382     Free(ctx, ctx);
       
   383 } // preprocessor_end
       
   384 
       
   385 
       
   386 const char *preprocessor_error(Preprocessor *_ctx)
       
   387 {
       
   388     Context *ctx = (Context *) _ctx;
       
   389     if (ctx->isfail)
       
   390     {
       
   391         ctx->isfail = 0;
       
   392         return ctx->failstr;
       
   393     } // if
       
   394 
       
   395     return NULL;
       
   396 } // preprocessor_error
       
   397 
       
   398 
       
   399 int preprocessor_outofmemory(Preprocessor *_ctx)
       
   400 {
       
   401     Context *ctx = (Context *) _ctx;
       
   402     return ctx->out_of_memory;
       
   403 } // preprocessor_outofmemory
       
   404 
       
   405 
       
   406 static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
       
   407                                              unsigned int *_len, Token *_token)
       
   408 {
       
   409     Context *ctx = (Context *) _ctx;
       
   410 
       
   411     while (1)
       
   412     {
       
   413         IncludeState *state = ctx->include_stack;
       
   414         if (state == NULL)
       
   415         {
       
   416             *_token = TOKEN_EOI;
       
   417             *_len = 0;
       
   418             return NULL;  // we're done!
       
   419         } // if
       
   420 
       
   421         Token token = preprocessor_internal_lexer(state);
       
   422         if (token == TOKEN_EOI)
       
   423         {
       
   424             assert(state->bytes_left == 0);
       
   425             pop_source(ctx);
       
   426             continue;  // pick up again after parent's #include line.
       
   427         } // if
       
   428 
       
   429         else if (token == TOKEN_PP_INCOMPLETE_COMMENT)
       
   430         {
       
   431             fail(ctx, "Incomplete multiline comment");
       
   432             continue;  // !!! FIXME: we should probably return TOKEN_ERROR or something.
       
   433         } // else if
       
   434 
       
   435         *_token = token;
       
   436         *_len = (unsigned int) (state->source - state->token);
       
   437         return state->token;
       
   438     } // while
       
   439 
       
   440     assert(0 && "shouldn't hit this code");
       
   441     *_token = TOKEN_UNKNOWN;
       
   442     *_len = 0;
       
   443     return NULL;
       
   444 } // _preprocessor_nexttoken
       
   445 
       
   446 
       
   447 const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,
       
   448                                    Token *token)
       
   449 {
       
   450     const char *retval = _preprocessor_nexttoken(ctx, len, token);
       
   451     print_debug_token(retval, *len, *token);
   443     return retval;
   452     return retval;
   444 } // preprocessor_nexttoken
   453 } // preprocessor_nexttoken
   445 
   454 
   446 
   455 
   447 const char *preprocessor_sourcepos(Preprocessor *_ctx, unsigned int *pos)
   456 const char *preprocessor_sourcepos(Preprocessor *_ctx, unsigned int *pos)