mojoshader_compiler.c
changeset 858 d51537335896
parent 857 905ad877371b
child 859 824d67791db0
equal deleted inserted replaced
857:905ad877371b 858:d51537335896
    32 {
    32 {
    33     int64 i64;
    33     int64 i64;
    34     double dbl;
    34     double dbl;
    35     const char *string;
    35     const char *string;
    36 } TokenData;
    36 } TokenData;
    37 
       
    38 typedef struct StringBucket
       
    39 {
       
    40     char *string;
       
    41     struct StringBucket *next;
       
    42 } StringBucket;
       
    43 
    37 
    44 
    38 
    45 // Structures that make up the parse tree...
    39 // Structures that make up the parse tree...
    46 
    40 
    47 typedef enum ASTNodeType
    41 typedef enum ASTNodeType
   545     MOJOSHADER_malloc malloc;
   539     MOJOSHADER_malloc malloc;
   546     MOJOSHADER_free free;
   540     MOJOSHADER_free free;
   547     void *malloc_data;
   541     void *malloc_data;
   548     int error_count;
   542     int error_count;
   549     ErrorList *errors;
   543     ErrorList *errors;
   550     StringBucket *string_hashtable[256];
   544     StringCache *strcache;
   551     const char *sourcefile;  // current source file that we're parsing.
   545     const char *sourcefile;  // current source file that we're parsing.
   552     unsigned int sourceline; // current line in sourcefile that we're parsing.
   546     unsigned int sourceline; // current line in sourcefile that we're parsing.
   553     UserTypeMap usertypes;
   547     UserTypeMap usertypes;
   554     CompilationUnit *ast;  // Abstract Syntax Tree
   548     CompilationUnit *ast;  // Abstract Syntax Tree
   555 } Context;
   549 } Context;
  1427 {
  1421 {
  1428     const void *value;
  1422     const void *value;
  1429     return hash_find(ctx->usertypes.types, token, &value);
  1423     return hash_find(ctx->usertypes.types, token, &value);
  1430 } // is_usertype
  1424 } // is_usertype
  1431 
  1425 
  1432 
       
  1433 // !!! FIXME: sort of cut-and-paste from the preprocessor...
       
  1434 
       
  1435 // this is djb's xor hashing function.
       
  1436 static inline uint32 hash_string_djbxor(const char *str, unsigned int len)
       
  1437 {
       
  1438     register uint32 hash = 5381;
       
  1439     while (len--)
       
  1440         hash = ((hash << 5) + hash) ^ *(str++);
       
  1441     return hash;
       
  1442 } // hash_string_djbxor
       
  1443 
       
  1444 static inline uint8 hash_string(const char *str, const unsigned int len)
       
  1445 {
       
  1446     return (uint8) hash_string_djbxor(str, len);
       
  1447 } // hash_string
       
  1448 
       
  1449 static const char *cache_string(Context *ctx, const char *str,
       
  1450                                 const unsigned int len)
       
  1451 {
       
  1452     const uint8 hash = hash_string(str, len);
       
  1453     StringBucket *bucket = ctx->string_hashtable[hash];
       
  1454     StringBucket *prev = NULL;
       
  1455     while (bucket)
       
  1456     {
       
  1457         const char *bstr = bucket->string;
       
  1458         if ((strncmp(bstr, str, len) == 0) && (bstr[len] == 0))
       
  1459         {
       
  1460             // Matched! Move this to the front of the list.
       
  1461             if (prev != NULL)
       
  1462             {
       
  1463                 assert(prev->next == bucket);
       
  1464                 prev->next = bucket->next;
       
  1465                 bucket->next = ctx->string_hashtable[hash];
       
  1466                 ctx->string_hashtable[hash] = bucket;
       
  1467             } // if
       
  1468             return bstr; // already cached
       
  1469         } // if
       
  1470         prev = bucket;
       
  1471         bucket = bucket->next;
       
  1472     } // while
       
  1473 
       
  1474     // no match, add to the table.
       
  1475     bucket = (StringBucket *) Malloc(ctx, sizeof (StringBucket));
       
  1476     if (bucket == NULL)
       
  1477         return NULL;
       
  1478     bucket->string = (char *) Malloc(ctx, len + 1);
       
  1479     if (bucket->string == NULL)
       
  1480     {
       
  1481         Free(ctx, bucket);
       
  1482         return NULL;
       
  1483     } // if
       
  1484     memcpy(bucket->string, str, len);
       
  1485     bucket->string[len] = '\0';
       
  1486     bucket->next = ctx->string_hashtable[hash];
       
  1487     ctx->string_hashtable[hash] = bucket;
       
  1488     return bucket->string;
       
  1489 } // cache_string
       
  1490 
       
  1491 static const char *cache_string_fmt(Context *ctx, const char *fmt, ...)
       
  1492 {
       
  1493     char buf[128];  // use the stack if reasonable.
       
  1494     char *ptr = NULL;
       
  1495     int len = 0;  // number of chars, NOT counting null-terminator!
       
  1496     va_list ap;
       
  1497 
       
  1498     va_start(ap, fmt);
       
  1499     len = vsnprintf(buf, sizeof (buf), fmt, ap);
       
  1500     va_end(ap);
       
  1501 
       
  1502     if (len > sizeof (buf))
       
  1503     {
       
  1504         ptr = (char *) Malloc(ctx, len);
       
  1505         if (ptr == NULL)
       
  1506             return NULL;
       
  1507 
       
  1508         va_start(ap, fmt);
       
  1509         vsnprintf(ptr, len, fmt, ap);
       
  1510         va_end(ap);
       
  1511     } // if
       
  1512 
       
  1513     const char *retval = cache_string(ctx, ptr ? ptr : buf, len);
       
  1514     if (ptr != NULL)
       
  1515         Free(ctx, ptr);
       
  1516 
       
  1517     return retval;
       
  1518 } // cache_string_fmt
       
  1519 
  1426 
  1520 // This is where the actual parsing happens. It's Lemon-generated!
  1427 // This is where the actual parsing happens. It's Lemon-generated!
  1521 #define __MOJOSHADER_HLSL_COMPILER__ 1
  1428 #define __MOJOSHADER_HLSL_COMPILER__ 1
  1522 #include "mojoshader_parser_hlsl.h"
  1429 #include "mojoshader_parser_hlsl.h"
  1523 
  1430 
  2432             if (tokencmp("xps")) return TOKEN_HLSL_XPS;
  2339             if (tokencmp("xps")) return TOKEN_HLSL_XPS;
  2433 
  2340 
  2434             #undef tokencmp
  2341             #undef tokencmp
  2435 
  2342 
  2436             // get a canonical copy of the string now, as we'll need it.
  2343             // get a canonical copy of the string now, as we'll need it.
  2437             token = cache_string(ctx, token, tokenlen);
  2344             token = stringcache_len(ctx->strcache, token, tokenlen);
  2438             if (is_usertype(ctx, token))
  2345             if (is_usertype(ctx, token))
  2439                 return TOKEN_HLSL_USERTYPE;
  2346                 return TOKEN_HLSL_USERTYPE;
  2440             return TOKEN_HLSL_IDENTIFIER;
  2347             return TOKEN_HLSL_IDENTIFIER;
  2441 
  2348 
  2442         case TOKEN_EOI: return 0;
  2349         case TOKEN_EOI: return 0;
  2443         default: assert(0 && "unexpected token from lexer\n"); return 0;
  2350         default: assert(0 && "unexpected token from lexer\n"); return 0;
  2444     } // switch
  2351     } // switch
  2445 
  2352 
  2446     return 0;
  2353     return 0;
  2447 } // convert_to_lemon_token
  2354 } // convert_to_lemon_token
  2448 
       
  2449 // !!! FIXME: unify this code with the string cache in the preprocessor.
       
  2450 static void free_string_cache(Context *ctx)
       
  2451 {
       
  2452     size_t i;
       
  2453     for (i = 0; i < STATICARRAYLEN(ctx->string_hashtable); i++)
       
  2454     {
       
  2455         StringBucket *bucket = ctx->string_hashtable[i];
       
  2456         ctx->string_hashtable[i] = NULL;
       
  2457         while (bucket)
       
  2458         {
       
  2459             StringBucket *next = bucket->next;
       
  2460             Free(ctx, bucket->string);
       
  2461             Free(ctx, bucket);
       
  2462             bucket = next;
       
  2463         } // while
       
  2464     } // for
       
  2465 } // free_string_cache
       
  2466 
  2355 
  2467 static void destroy_context(Context *ctx)
  2356 static void destroy_context(Context *ctx)
  2468 {
  2357 {
  2469     if (ctx != NULL)
  2358     if (ctx != NULL)
  2470     {
  2359     {
  2472         void *d = ctx->malloc_data;
  2361         void *d = ctx->malloc_data;
  2473 
  2362 
  2474         // !!! FIXME: free ctx->errors
  2363         // !!! FIXME: free ctx->errors
  2475         delete_compilation_unit(ctx, ctx->ast);
  2364         delete_compilation_unit(ctx, ctx->ast);
  2476         destroy_usertypemap(ctx);
  2365         destroy_usertypemap(ctx);
  2477         free_string_cache(ctx);
  2366 
       
  2367         if (ctx->strcache)
       
  2368             stringcache_destroy(ctx->strcache);
       
  2369 
  2478         f(ctx, d);
  2370         f(ctx, d);
  2479     } // if
  2371     } // if
  2480 } // destroy_context
  2372 } // destroy_context
  2481 
  2373 
  2482 static Context *build_context(MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
  2374 static Context *build_context(MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
  2492     ctx->malloc = m;
  2384     ctx->malloc = m;
  2493     ctx->free = f;
  2385     ctx->free = f;
  2494     ctx->malloc_data = d;
  2386     ctx->malloc_data = d;
  2495     //ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED;
  2387     //ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED;
  2496     create_usertypemap(ctx);  // !!! FIXME: check for failure.
  2388     create_usertypemap(ctx);  // !!! FIXME: check for failure.
       
  2389     ctx->strcache = stringcache_create(m, f, d);  // !!! FIXME: check for failure.
       
  2390 
  2497     return ctx;
  2391     return ctx;
  2498 } // build_context
  2392 } // build_context
  2499 
  2393 
  2500 
  2394 
  2501 // parse the source code into an AST.
  2395 // parse the source code into an AST.
  2539     {
  2433     {
  2540         char buf[32];
  2434         char buf[32];
  2541         int j;
  2435         int j;
  2542         for (j = 1; j <= 4; j++)
  2436         for (j = 1; j <= 4; j++)
  2543         {
  2437         {
       
  2438             // "float2"
  2544             int len = snprintf(buf, sizeof (buf), "%s%d", types[i], j);
  2439             int len = snprintf(buf, sizeof (buf), "%s%d", types[i], j);
  2545             add_usertype(ctx, cache_string(ctx, buf, len));  // "float2"
  2440             add_usertype(ctx, stringcache_len(ctx->strcache, buf, len));
  2546             int k;
  2441             int k;
  2547             for (k = 1; k <= 4; k++)
  2442             for (k = 1; k <= 4; k++)
  2548             {
  2443             {
       
  2444                 // "float2x2"
  2549                 len = snprintf(buf, sizeof (buf), "%s%dx%d", types[i], j, k);
  2445                 len = snprintf(buf, sizeof (buf), "%s%dx%d", types[i], j, k);
  2550                 add_usertype(ctx, cache_string(ctx, buf, len));  // "float2x2"
  2446                 add_usertype(ctx, stringcache_len(ctx->strcache, buf, len));
  2551             } // for
  2447             } // for
  2552         } // for
  2448         } // for
  2553     } // for
  2449     } // for
  2554 
  2450 
  2555     // Run the preprocessor/lexer/parser...
  2451     // Run the preprocessor/lexer/parser...
  2561             out_of_memory(ctx);
  2457             out_of_memory(ctx);
  2562             break;
  2458             break;
  2563         } // if
  2459         } // if
  2564 
  2460 
  2565         fname = preprocessor_sourcepos(pp, &ctx->sourceline);
  2461         fname = preprocessor_sourcepos(pp, &ctx->sourceline);
  2566         ctx->sourcefile = fname ? cache_string(ctx, fname, strlen(fname)) : 0;
  2462         ctx->sourcefile = fname ? stringcache(ctx->strcache, fname) : 0;
  2567 
  2463 
  2568         if (tokenval == TOKEN_BAD_CHARS)
  2464         if (tokenval == TOKEN_BAD_CHARS)
  2569         {
  2465         {
  2570             fail(ctx, "Bad characters in source file");
  2466             fail(ctx, "Bad characters in source file");
  2571             continue;
  2467             continue;
  2589                 break;
  2485                 break;
  2590 
  2486 
  2591             case TOKEN_HLSL_USERTYPE:
  2487             case TOKEN_HLSL_USERTYPE:
  2592             case TOKEN_HLSL_STRING_LITERAL:
  2488             case TOKEN_HLSL_STRING_LITERAL:
  2593             case TOKEN_HLSL_IDENTIFIER:
  2489             case TOKEN_HLSL_IDENTIFIER:
  2594                 data.string = cache_string(ctx, token, tokenlen);
  2490                 data.string = stringcache_len(ctx->strcache, token, tokenlen);
  2595                 break;
  2491                 break;
  2596 
  2492 
  2597             default:
  2493             default:
  2598                 data.i64 = 0;
  2494                 data.i64 = 0;
  2599                 break;
  2495                 break;