13 #define DEBUG_TOKENIZER 0

    13 #if DEBUG_PREPROCESSOR

    14     #define print_debug_token(token, len, val) \

    15         MOJOSHADER_print_debug_token("PREPROCESSOR", token, len, val)

    16 #else

    17     #define print_debug_token(token, len, val)

    18 #endif

    84 // Preprocessor define hashtable stuff...

    89 #if DEBUG_TOKENIZER

    85 

    90 void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,

    86 static unsigned char hash_define(const char *sym)

    91                                   const unsigned int tokenlen,

    87 {

    92                                   const Token tokenval)

    88     unsigned char retval = 0;

    93 {

    89     while (sym)

    94     printf("%s TOKEN: \"", subsystem);

    90         retval += *(sym++);

    91     return retval;

    92 } // hash_define

    93 

    94 

    95 static int add_define(Context *ctx, const char *sym, const char *val)

    96 {

    97     char *identifier = NULL;

    98     char *definition = NULL;

    99     const unsigned char hash = hash_define(sym);

   100     DefineHash *bucket = ctx->define_hashtable[hash];

   101     while (bucket)

   102     {

   103         if (strcmp(bucket->define.identifier, sym) == 0)

   104         {

   105             failf(ctx, "'%s' already defined", sym);

   106             return 0;

   107         } // if

   108         bucket = bucket->next;

   109     } // while

   110 

   111     bucket = (DefineHash *) Malloc(ctx, sizeof (DefineHash));

   112     if (bucket == NULL)

   113         return 0;

   114 

   115     identifier = (char *) Malloc(ctx, strlen(sym) + 1);

   116     definition = (char *) Malloc(ctx, strlen(val) + 1);

   117     if ((identifier == NULL) || (definition == NULL))

   118     {

   119         Free(ctx, identifier);

   120         Free(ctx, definition);

   121         Free(ctx, bucket);

   122         return 0;

   123     } // if

   124 

   125     strcpy(identifier, sym);

   126     strcpy(definition, val);

   127     bucket->define.identifier = identifier;

   128     bucket->define.definition = definition;

   129     bucket->next = ctx->define_hashtable[hash];

   130     ctx->define_hashtable[hash] = bucket;

   131     return 1;

   132 } // add_define

   133 

   134 

   135 static int remove_define(Context *ctx, const char *sym)

   136 {

   137     const unsigned char hash = hash_define(sym);

   138     DefineHash *bucket = ctx->define_hashtable[hash];

   139     DefineHash *prev = NULL;

   140     while (bucket)

   141     {

   142         if (strcmp(bucket->define.identifier, sym) == 0)

   143         {

   144             if (prev == NULL)

   145                 ctx->define_hashtable[hash] = bucket->next;

   146             else

   147                 prev->next = bucket->next;

   148             Free(ctx, (void *) bucket->define.identifier);

   149             Free(ctx, (void *) bucket->define.definition);

   150             Free(ctx, bucket);

   151             return 1;

   152         } // if

   153         prev = bucket;

   154         bucket = bucket->next;

   155     } // while

   156 

   157     failf(ctx, "'%s' not defined", sym);

   158     return 0;

   159 } // remove_define

   160 

   161 

   162 static const char *find_define(Context *ctx, const char *sym)

   163 {

   164     const unsigned char hash = hash_define(sym);

   165     DefineHash *bucket = ctx->define_hashtable[hash];

   166     while (bucket)

   167     {

   168         if (strcmp(bucket->define.identifier, sym) == 0)

   169             return bucket->define.definition;

   170         bucket = bucket->next;

   171     } // while

   172     return NULL;

   173 } // find_define

   174 

   175 

   176 static void free_all_defines(Context *ctx)

   177 {

   178     int i;

   179     for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)

   180     {

   181         DefineHash *bucket = ctx->define_hashtable[i];

   182         ctx->define_hashtable[i] = NULL;

   183         while (bucket)

   184         {

   185             DefineHash *next = bucket->next;

   186             Free(ctx, (void *) bucket->define.identifier);

   187             Free(ctx, (void *) bucket->define.definition);

   188             Free(ctx, bucket);

   189             bucket = next;

   190         } // while

   191     } // for

   192 } // find_define

   193 

   194 

   195 static int push_source(Context *ctx, const char *fname, const char *source,

   196                        unsigned int srclen, int included)

   197 {

   198     IncludeState *state = (IncludeState *) Malloc(ctx, sizeof (IncludeState));

   199     if (state == NULL)

   200         return 0;

   201     memset(state, '\0', sizeof (IncludeState));

   202 

   203     if (fname != NULL)

   204     {

   205         state->filename = StrDup(ctx, fname);

   206         if (state->filename == NULL)

   207         {

   208             Free(ctx, state);

   209             return 0;

   210         } // if

   211     } // if

   212 

   213     state->included = included;

   214     state->source_base = source;

   215     state->source = source;

   216     state->token = source;

   217     state->bytes_left = srclen;

   218     state->line = 1;

   219     state->next = ctx->include_stack;

   220 

   221     ctx->include_stack = state;

   222 

   223     return 1;

   224 } // push_source

   225 

   226 

   227 static void pop_source(Context *ctx)

   228 {

   229     IncludeState *state = ctx->include_stack;

   230     if (state == NULL)

   231         return;

   232 

   233     if (state->included)

   234     {

   235         ctx->close_callback(state->source_base, ctx->malloc,

   236                             ctx->free, ctx->malloc_data);

   237     } // if

   238 

   239     ctx->include_stack = state->next;

   240     Free(ctx, state->filename);

   241     Free(ctx, state);

   242 } // pop_source

   243 

   244 

   245 Preprocessor *preprocessor_start(const char *fname, const char *source,

   246                             unsigned int sourcelen,

   247                             MOJOSHADER_includeOpen open_callback,

   248                             MOJOSHADER_includeClose close_callback,

   249                             const MOJOSHADER_preprocessorDefine **defines,

   250                             unsigned int define_count,

   251                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)

   252 {

   253     int okay = 1;

   254     int i = 0;

   255 

   256     // the preprocessor is internal-only, so we verify all these are != NULL.

   257     assert(m != NULL);

   258     assert(f != NULL);

   259     assert(open_callback != NULL);

   260     assert(close_callback != NULL);

   261 

   262     Context *ctx = (Context *) m(sizeof (Context), d);

   263     if (ctx == NULL)

   264         return 0;

   265 

   266     memset(ctx, '\0', sizeof (Context));

   267     ctx->malloc = m;

   268     ctx->free = f;

   269     ctx->malloc_data = d;

   270     ctx->open_callback = open_callback;

   271     ctx->close_callback = close_callback;

   272 

   273     for (i = 0; i < define_count; i++)

   274     {

   275         if (!add_define(ctx, defines[i]->identifier, defines[i]->definition))

   276         {

   277             okay = 0;

   278             break;

   279         } // if

   280     } // for

   281 

   282     if ((okay) && (!push_source(ctx, fname, source, sourcelen, 0)))

   283         okay = 0;

   284 

   285     if (!okay)

   286     {

   287         preprocessor_end((Preprocessor *) ctx);

   288         return NULL;

   289     } // if

   290 

   291     return (Preprocessor *) ctx;

   292 } // preprocessor_start

   293 

   294 

   295 void preprocessor_end(Preprocessor *_ctx)

   296 {

   297     Context *ctx = (Context *) _ctx;

   298     if (ctx == NULL)

   299         return;

   300 

   301     while (ctx->include_stack != NULL)

   302         pop_source(ctx);

   303 

   304     free_all_defines(ctx);

   305 

   306     Free(ctx, ctx);

   307 } // preprocessor_end

   308 

   309 

   310 const char *preprocessor_error(Preprocessor *_ctx)

   311 {

   312     Context *ctx = (Context *) _ctx;

   313     if (ctx->isfail)

   314     {

   315         ctx->isfail = 0;

   316         return ctx->failstr;

   317     } // if

   318 

   319     return NULL;

   320 } // preprocessor_error

   321 

   322 

   323 int preprocessor_outofmemory(Preprocessor *_ctx)

   324 {

   325     Context *ctx = (Context *) _ctx;

   326     return ctx->out_of_memory;

   327 } // preprocessor_outofmemory

   328 

   329 

   330 static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,

   331                                              unsigned int *_len, Token *_token)

   332 {

   333     Context *ctx = (Context *) _ctx;

   334 

   335     while (1)

   336     {

   337         IncludeState *state = ctx->include_stack;

   338         if (state == NULL)

   339         {

   340             *_token = TOKEN_EOI;

   341             *_len = 0;

   342             return NULL;  // we're done!

   343         } // if

   344 

   345         Token token = preprocessor_internal_lexer(state);

   346         if (token == TOKEN_EOI)

   347         {

   348             assert(state->bytes_left == 0);

   349             pop_source(ctx);

   350             continue;  // pick up again after parent's #include line.

   351         } // if

   352 

   353         else if (token == TOKEN_PP_INCOMPLETE_COMMENT)

   354         {

   355             fail(ctx, "Incomplete multiline comment");

   356             continue;  // !!! FIXME: we should probably return TOKEN_ERROR or something.

   357         } // else if

   358 

   359         *_token = token;

   360         *_len = (unsigned int) (state->source - state->token);

   361         return state->token;

   362     } // while

   363 

   364     assert(0 && "shouldn't hit this code");

   365     *_token = TOKEN_UNKNOWN;

   366     *_len = 0;

   367     return NULL;

   368 } // _preprocessor_nexttoken

   369 

   370 

   371 const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,

   372                                    Token *token)

   373 {

   374     const char *retval = _preprocessor_nexttoken(ctx, len, token);

   375 

   376     #if DEBUG_TOKENIZER

   377     printf("PREPROCESSOR TOKEN: \"");

   379     for (i = 0; i < *len; i++)

    96     for (i = 0; i < tokenlen; i++)

   380     {

    97     {

   381         if (retval[i] == '\n')

    98         if (token[i] == '\n')

   384             printf("%c", retval[i]);

   101             printf("%c", token[i]);

   387     switch (*token)

   104     switch (tokenval)

   436             assert(((int)*token) < 256);

   151             assert(((int)tokenval) < 256);

   437             printf("'%c'", (char) *token);

   152             printf("'%c'", (char) tokenval);

   441     #endif

   156 } // MOJOSHADER_print_debug_token

   442 

   157 #endif

   158 

   159 

   160 // Preprocessor define hashtable stuff...

   161 

   162 static unsigned char hash_define(const char *sym)

   163 {

   164     unsigned char retval = 0;

   165     while (sym)

   166         retval += *(sym++);

   167     return retval;

   168 } // hash_define

   169 

   170 

   171 static int add_define(Context *ctx, const char *sym, const char *val)

   172 {

   173     char *identifier = NULL;

   174     char *definition = NULL;

   175     const unsigned char hash = hash_define(sym);

   176     DefineHash *bucket = ctx->define_hashtable[hash];

   177     while (bucket)

   178     {

   179         if (strcmp(bucket->define.identifier, sym) == 0)

   180         {

   181             failf(ctx, "'%s' already defined", sym);

   182             return 0;

   183         } // if

   184         bucket = bucket->next;

   185     } // while

   186 

   187     bucket = (DefineHash *) Malloc(ctx, sizeof (DefineHash));

   188     if (bucket == NULL)

   189         return 0;

   190 

   191     identifier = (char *) Malloc(ctx, strlen(sym) + 1);

   192     definition = (char *) Malloc(ctx, strlen(val) + 1);

   193     if ((identifier == NULL) || (definition == NULL))

   194     {

   195         Free(ctx, identifier);

   196         Free(ctx, definition);

   197         Free(ctx, bucket);

   198         return 0;

   199     } // if

   200 

   201     strcpy(identifier, sym);

   202     strcpy(definition, val);

   203     bucket->define.identifier = identifier;

   204     bucket->define.definition = definition;

   205     bucket->next = ctx->define_hashtable[hash];

   206     ctx->define_hashtable[hash] = bucket;

   207     return 1;

   208 } // add_define

   209 

   210 

   211 static int remove_define(Context *ctx, const char *sym)

   212 {

   213     const unsigned char hash = hash_define(sym);

   214     DefineHash *bucket = ctx->define_hashtable[hash];

   215     DefineHash *prev = NULL;

   216     while (bucket)

   217     {

   218         if (strcmp(bucket->define.identifier, sym) == 0)

   219         {

   220             if (prev == NULL)

   221                 ctx->define_hashtable[hash] = bucket->next;

   222             else

   223                 prev->next = bucket->next;

   224             Free(ctx, (void *) bucket->define.identifier);

   225             Free(ctx, (void *) bucket->define.definition);

   226             Free(ctx, bucket);

   227             return 1;

   228         } // if

   229         prev = bucket;

   230         bucket = bucket->next;

   231     } // while

   232 

   233     failf(ctx, "'%s' not defined", sym);

   234     return 0;

   235 } // remove_define

   236 

   237 

   238 static const char *find_define(Context *ctx, const char *sym)

   239 {

   240     const unsigned char hash = hash_define(sym);

   241     DefineHash *bucket = ctx->define_hashtable[hash];

   242     while (bucket)

   243     {

   244         if (strcmp(bucket->define.identifier, sym) == 0)

   245             return bucket->define.definition;

   246         bucket = bucket->next;

   247     } // while

   248     return NULL;

   249 } // find_define

   250 

   251 

   252 static void free_all_defines(Context *ctx)

   253 {

   254     int i;

   255     for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)

   256     {

   257         DefineHash *bucket = ctx->define_hashtable[i];

   258         ctx->define_hashtable[i] = NULL;

   259         while (bucket)

   260         {

   261             DefineHash *next = bucket->next;

   262             Free(ctx, (void *) bucket->define.identifier);

   263             Free(ctx, (void *) bucket->define.definition);

   264             Free(ctx, bucket);

   265             bucket = next;

   266         } // while

   267     } // for

   268 } // find_define

   269 

   270 

   271 static int push_source(Context *ctx, const char *fname, const char *source,

   272                        unsigned int srclen, int included)

   273 {

   274     IncludeState *state = (IncludeState *) Malloc(ctx, sizeof (IncludeState));

   275     if (state == NULL)

   276         return 0;

   277     memset(state, '\0', sizeof (IncludeState));

   278 

   279     if (fname != NULL)

   280     {

   281         state->filename = StrDup(ctx, fname);

   282         if (state->filename == NULL)

   283         {

   284             Free(ctx, state);

   285             return 0;

   286         } // if

   287     } // if

   288 

   289     state->included = included;

   290     state->source_base = source;

   291     state->source = source;

   292     state->token = source;

   293     state->bytes_left = srclen;

   294     state->line = 1;

   295     state->next = ctx->include_stack;

   296 

   297     ctx->include_stack = state;

   298 

   299     return 1;

   300 } // push_source

   301 

   302 

   303 static void pop_source(Context *ctx)

   304 {

   305     IncludeState *state = ctx->include_stack;

   306     if (state == NULL)

   307         return;

   308 

   309     if (state->included)

   310     {

   311         ctx->close_callback(state->source_base, ctx->malloc,

   312                             ctx->free, ctx->malloc_data);

   313     } // if

   314 

   315     ctx->include_stack = state->next;

   316     Free(ctx, state->filename);

   317     Free(ctx, state);

   318 } // pop_source

   319 

   320 

   321 Preprocessor *preprocessor_start(const char *fname, const char *source,

   322                             unsigned int sourcelen,

   323                             MOJOSHADER_includeOpen open_callback,

   324                             MOJOSHADER_includeClose close_callback,

   325                             const MOJOSHADER_preprocessorDefine **defines,

   326                             unsigned int define_count,

   327                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)

   328 {

   329     int okay = 1;

   330     int i = 0;

   331 

   332     // the preprocessor is internal-only, so we verify all these are != NULL.

   333     assert(m != NULL);

   334     assert(f != NULL);

   335     assert(open_callback != NULL);

   336     assert(close_callback != NULL);

   337 

   338     Context *ctx = (Context *) m(sizeof (Context), d);

   339     if (ctx == NULL)

   340         return 0;

   341 

   342     memset(ctx, '\0', sizeof (Context));

   343     ctx->malloc = m;

   344     ctx->free = f;

   345     ctx->malloc_data = d;

   346     ctx->open_callback = open_callback;

   347     ctx->close_callback = close_callback;

   348 

   349     for (i = 0; i < define_count; i++)

   350     {

   351         if (!add_define(ctx, defines[i]->identifier, defines[i]->definition))

   352         {

   353             okay = 0;

   354             break;

   355         } // if

   356     } // for

   357 

   358     if ((okay) && (!push_source(ctx, fname, source, sourcelen, 0)))

   359         okay = 0;

   360 

   361     if (!okay)

   362     {

   363         preprocessor_end((Preprocessor *) ctx);

   364         return NULL;

   365     } // if

   366 

   367     return (Preprocessor *) ctx;

   368 } // preprocessor_start

   369 

   370 

   371 void preprocessor_end(Preprocessor *_ctx)

   372 {

   373     Context *ctx = (Context *) _ctx;

   374     if (ctx == NULL)

   375         return;

   376 

   377     while (ctx->include_stack != NULL)

   378         pop_source(ctx);

   379 

   380     free_all_defines(ctx);

   381 

   382     Free(ctx, ctx);

   383 } // preprocessor_end

   384 

   385 

   386 const char *preprocessor_error(Preprocessor *_ctx)

   387 {

   388     Context *ctx = (Context *) _ctx;

   389     if (ctx->isfail)

   390     {

   391         ctx->isfail = 0;

   392         return ctx->failstr;

   393     } // if

   394 

   395     return NULL;

   396 } // preprocessor_error

   397 

   398 

   399 int preprocessor_outofmemory(Preprocessor *_ctx)

   400 {

   401     Context *ctx = (Context *) _ctx;

   402     return ctx->out_of_memory;

   403 } // preprocessor_outofmemory

   404 

   405 

   406 static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,

   407                                              unsigned int *_len, Token *_token)

   408 {

   409     Context *ctx = (Context *) _ctx;

   410 

   411     while (1)

   412     {

   413         IncludeState *state = ctx->include_stack;

   414         if (state == NULL)

   415         {

   416             *_token = TOKEN_EOI;

   417             *_len = 0;

   418             return NULL;  // we're done!

   419         } // if

   420 

   421         Token token = preprocessor_internal_lexer(state);

   422         if (token == TOKEN_EOI)

   423         {

   424             assert(state->bytes_left == 0);

   425             pop_source(ctx);

   426             continue;  // pick up again after parent's #include line.

   427         } // if

   428 

   429         else if (token == TOKEN_PP_INCOMPLETE_COMMENT)

   430         {

   431             fail(ctx, "Incomplete multiline comment");

   432             continue;  // !!! FIXME: we should probably return TOKEN_ERROR or something.

   433         } // else if

   434 

   435         *_token = token;

   436         *_len = (unsigned int) (state->source - state->token);

   437         return state->token;

   438     } // while

   439 

   440     assert(0 && "shouldn't hit this code");

   441     *_token = TOKEN_UNKNOWN;

   442     *_len = 0;

   443     return NULL;

   444 } // _preprocessor_nexttoken

   445 

   446 

   447 const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,

   448                                    Token *token)

   449 {

   450     const char *retval = _preprocessor_nexttoken(ctx, len, token);

   451     print_debug_token(retval, *len, *token);