First (incomplete!) shot at more robust CTAB support.
authorRyan C. Gordon <icculus@icculus.org>
Sat, 20 Dec 2008 05:48:38 -0500
changeset 524 03eea2f0762c
parent 523 699696afd731
child 525 0493d39de79a
First (incomplete!) shot at more robust CTAB support. Assembler can now add it, but we still need to parse it better and report it all back to the app in MOJOSHADER_parse().
assemble.c
finderrors.c
mojoshader.c
mojoshader.h
mojoshader_assembler.c
mojoshader_internal.h
--- a/assemble.c	Fri Dec 19 00:05:20 2008 -0500
+++ b/assemble.c	Sat Dec 20 05:48:38 2008 -0500
@@ -23,7 +23,7 @@
     const MOJOSHADER_parseData *pd;
     int retval = 0;
 
-    pd = MOJOSHADER_assemble(buf, NULL, NULL, NULL);
+    pd = MOJOSHADER_assemble(buf, NULL, 0, NULL, 0, NULL, NULL, NULL);
     if (pd->error != NULL)
         printf("ERROR: (line %d) %s\n", pd->error_position, pd->error);
     else
--- a/finderrors.c	Fri Dec 19 00:05:20 2008 -0500
+++ b/finderrors.c	Sat Dec 20 05:48:38 2008 -0500
@@ -84,7 +84,7 @@
         const MOJOSHADER_parseData *a;
 
         buf[rc] = '\0';  // make sure the source is null-terminated.
-        a = MOJOSHADER_assemble((char *) buf, 0, 0, 0);
+        a = MOJOSHADER_assemble((char *) buf, 0, 0, 0, 0, 0, 0, 0);
         if (a->error)
         {
             report("FAIL: %s (line %d) %s\n", fname, a->error_position, a->error);
--- a/mojoshader.c	Fri Dec 19 00:05:20 2008 -0500
+++ b/mojoshader.c	Sat Dec 20 05:48:38 2008 -0500
@@ -5737,6 +5737,7 @@
     return item;
 } // alloc_constant_listitem
 
+
 static void state_DEF(Context *ctx)
 {
     const RegisterType regtype = ctx->dest_arg.regtype;
@@ -6517,24 +6518,22 @@
     const uint32 target = SWAP32(ctx->tokens[8]);
     uint32 i = 0;
 
-    if (id != 0x42415443)  // 0x42415443 == 'CTAB'
+    if (id != CTAB_ID)
         return;  // not the constant table.
 
-    if (size != 28)
+    if (size != CTAB_SIZE)
         return;  // only handle this version of the struct.
 
     if (version != ctx->version_token) goto corrupt_ctab;
     if (creator >= bytes) goto corrupt_ctab;
-    if ((constantinfo + (constants * 20)) >= bytes) goto corrupt_ctab;
+    if ((constantinfo + (constants * CINFO_SIZE)) >= bytes) goto corrupt_ctab;
     if (target >= bytes) goto corrupt_ctab;
 
     ctx->have_ctab = 1;
 
     for (i = 0; i < constants; i++)
     {
-        // we only care about deciding which variables might be arrays at
-        //  the moment, but there's lots of other good info in the CTAB.
-        const uint8 *ptr = start + constantinfo + (i * 20);
+        const uint8 *ptr = start + constantinfo + (i * CINFO_SIZE);
         const uint32 name = SWAP32(*((uint32 *) (ptr + 0)));
         const uint16 regset = SWAP16(*((uint16 *) (ptr + 4)));
         const uint16 regidx = SWAP16(*((uint16 *) (ptr + 6)));
@@ -7471,6 +7470,18 @@
         f((void *) data->samplers, d);
     } // if
 
+    if (data->symbols != NULL)
+    {
+        for (i = 0; i < data->symbol_count; i++)
+        {
+            if (data->symbols[i].name != NULL)
+                f((void *) data->symbols[i].name, d);
+            if (data->symbols[i].default_value != NULL)
+                f((void *) data->symbols[i].default_value, d);
+        } // for
+        f((void *) data->symbols, d);
+    } // if
+
     if ((data->error != NULL) && (data->error != out_of_mem_str))
         f((void *) data->error, d);
 
--- a/mojoshader.h	Fri Dec 19 00:05:20 2008 -0500
+++ b/mojoshader.h	Sat Dec 20 05:48:38 2008 -0500
@@ -240,8 +240,89 @@
 
 
 /*
+ * MOJOSHADER_symbol data.
+ *
+ * These are used to expose high-level information in shader bytecode.
+ *  They associate HLSL variables with registers. This data is used for both
+ *  debugging and optimization.
+ */
+
+typedef enum
+{
+    MOJOSHADER_SYMREGSET_BOOL,
+    MOJOSHADER_SYMREGSET_INT4,
+    MOJOSHADER_SYMREGSET_FLOAT4,
+    MOJOSHADER_SYMREGSET_SAMPLER,
+} MOJOSHADER_symbolRegisterSet;
+
+typedef enum
+{
+    MOJOSHADER_SYMCLASS_SCALAR,
+    MOJOSHADER_SYMCLASS_VECTOR,
+    MOJOSHADER_SYMCLASS_MATRIX_ROWS,
+    MOJOSHADER_SYMCLASS_MATRIX_COLUMNS,
+    MOJOSHADER_SYMCLASS_OBJECT,
+    MOJOSHADER_SYMCLASS_STRUCT,
+} MOJOSHADER_symbolClass;
+
+typedef enum
+{
+    MOJOSHADER_SYMTYPE_VOID,
+    MOJOSHADER_SYMTYPE_BOOL,
+    MOJOSHADER_SYMTYPE_INT,
+    MOJOSHADER_SYMTYPE_FLOAT,
+    MOJOSHADER_SYMTYPE_STRING,
+    MOJOSHADER_SYMTYPE_TEXTURE,
+    MOJOSHADER_SYMTYPE_TEXTURE1D,
+    MOJOSHADER_SYMTYPE_TEXTURE2D,
+    MOJOSHADER_SYMTYPE_TEXTURE3D,
+    MOJOSHADER_SYMTYPE_TEXTURECUBE,
+    MOJOSHADER_SYMTYPE_SAMPLER,
+    MOJOSHADER_SYMTYPE_SAMPLER1D,
+    MOJOSHADER_SYMTYPE_SAMPLER2D,
+    MOJOSHADER_SYMTYPE_SAMPLER3D,
+    MOJOSHADER_SYMTYPE_SAMPLERCUBE,
+    MOJOSHADER_SYMTYPE_PIXELSHADER,
+    MOJOSHADER_SYMTYPE_VERTEXSHADER,
+    MOJOSHADER_SYMTYPE_PIXELFRAGMENT,
+    MOJOSHADER_SYMTYPE_VERTEXFRAGMENT,
+    MOJOSHADER_SYMTYPE_UNSUPPORTED,
+} MOJOSHADER_symbolType;
+
+typedef struct MOJOSHADER_symbolStructMember MOJOSHADER_symbolStructMember;
+
+typedef struct MOJOSHADER_symbolTypeInfo
+{
+    MOJOSHADER_symbolClass parameter_class;
+    MOJOSHADER_symbolType parameter_type;
+    unsigned int rows;
+    unsigned int columns;
+    unsigned int elements;
+    unsigned int member_count;
+    MOJOSHADER_symbolStructMember *members;
+} MOJOSHADER_symbolTypeInfo;
+
+struct MOJOSHADER_symbolStructMember
+{
+    const char *name;
+    MOJOSHADER_symbolTypeInfo info;
+};
+
+typedef struct MOJOSHADER_symbol
+{
+    const char *name;
+    MOJOSHADER_symbolRegisterSet register_set;
+    unsigned int register_index;
+    unsigned int register_count;
+    MOJOSHADER_symbolTypeInfo info;
+    void *default_value;
+} MOJOSHADER_symbol;
+
+
+/*
  * Structure used to return data from parsing of a shader...
  */
+/* !!! FIXME: most of these ints should be unsigned. */
 typedef struct MOJOSHADER_parseData
 {
     /*
@@ -373,6 +454,20 @@
     MOJOSHADER_swizzle *swizzles;
 
     /*
+     * The number of elements pointed to by (symbols).
+     */
+    int symbol_count;
+
+    /*
+     * (symbol_count) elements of data that specify high-level symbol data
+     *  for the shader. This will be parsed from the CTAB section
+     *  in bytecode, and will be a copy of what you provide to
+     *  MOJOSHADER_assemble(). This data is optional.
+     * This can be NULL on error or if (symbol_count) is zero.
+     */
+    MOJOSHADER_symbol *symbols;
+
+    /*
      * This is the malloc implementation you passed to MOJOSHADER_parse().
      */
     MOJOSHADER_malloc malloc;
@@ -504,6 +599,17 @@
  * (source) is an ASCII, NULL-terminated string of valid Direct3D shader
  *  assembly source code.
  *
+ * (comments) points to (comment_count) NULL-terminated ASCII strings, and
+ *  can be NULL. These strings are inserted as comments in the bytecode.
+ *
+ * (symbols) points to (symbol_count) symbol structs, and can be NULL. These
+ *  become a CTAB field in the bytecode. This is optional, but
+ *  MOJOSHADER_parse() needs CTAB data for all arrays used in a program, or
+ *  relative addressing will not be permitted, so you'll want to at least
+ *  provide symbol information for those. The symbol data is 100% trusted
+ *  at this time; it will not be checked to see if it matches what was
+ *  assembled in any way whatsoever.
+ *
  * This will return a MOJOSHADER_parseData(), like MOJOSHADER_parse() would,
  *  except the profile will be MOJOSHADER_PROFILE_BYTECODE and the output
  *  will be the assembled bytecode instead of some other language. This output
@@ -526,7 +632,10 @@
  *  to assemble several shaders on separate CPU cores at the same time.
  */
 const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *source,
-                              MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
+                             const char **comments, unsigned int comment_count,
+                             const MOJOSHADER_symbol *symbols,
+                             unsigned int symbol_count,
+                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
 
 
 
--- a/mojoshader_assembler.c	Fri Dec 19 00:05:20 2008 -0500
+++ b/mojoshader_assembler.c	Sat Dec 20 05:48:38 2008 -0500
@@ -39,11 +39,15 @@
     MOJOSHADER_shaderType shader_type;
     uint8 major_ver;
     uint8 minor_ver;
+    uint32 version_token;
     uint32 tokenbuf[16];
     int tokenbufpos;
     DestArgInfo dest_arg;
     uint32 *output;
     uint32 *token_to_line;
+    uint8 *ctab;
+    uint32 ctab_len;
+    uint32 ctab_allocation;
     size_t output_len;
     size_t output_allocation;
 };
@@ -1568,7 +1572,8 @@
     if (require_endline(ctx) == FAIL)
         return FAIL;
 
-    output_token(ctx, (shader_type << 16) | (major << 8) | (minor << 0) );
+    ctx->version_token = (shader_type << 16) | (major << 8) | (minor << 0);
+    output_token(ctx, ctx->version_token);
     return NOFAIL;
 } // parse_version_token
 
@@ -1640,6 +1645,8 @@
             f(ctx->output, d);
         if (ctx->token_to_line != NULL)
             f(ctx->token_to_line, d);
+        if (ctx->ctab != NULL)
+            f(ctx->ctab, d);
         f(ctx, d);
     } // if
 } // destroy_context
@@ -1679,97 +1686,144 @@
 } // build_failed_assembly
 
 
-typedef struct CTabTypeInfo
+static uint32 add_ctab_bytes(Context *ctx, const uint8 *bytes, const size_t len)
 {
-    uint16 parameter_class;
-    uint16 parameter_type;
-    uint16 rows;
-    uint16 columns;
-    uint16 elements;
-    uint16 structMembers;
-    uint32 structMemberInfo;
-} CTabTypeInfo;
+    const size_t extra = CTAB_SIZE + sizeof (uint32);
+    if (len <= (ctx->ctab_len - extra))
+    {
+        void *ptr = ctx->ctab + extra;
+        while ((ptr = memchr(ptr, bytes[0], ctx->ctab_len - len)) != NULL)
+        {
+            if (memcmp(ptr, bytes, len) == 0)  // already have it?
+                return ( (uint32) (ctx->ctab - ((uint8 *) ptr)) );
+            ptr++;
+        } // while
+    } // if
+
+    // add it to the byte pile...
+
+    // verify allocation.
+    const size_t newsize = (ctx->ctab_len + len);
+    if (ctx->ctab_allocation < newsize)
+    {
+        const size_t additional = 4 * 1024;
+        while (ctx->ctab_allocation < newsize)
+            ctx->ctab_allocation += additional;
+        void *ptr = Malloc(ctx, ctx->ctab_allocation);
+        if (ptr == NULL)
+            return 0;
+        memcpy(ptr, ctx->ctab, ctx->ctab_len);
+        Free(ctx, ctx->ctab);
+        ctx->ctab = (uint8 *) ptr;
+    } // if
+
+    const uint32 retval = ctx->ctab_len;
+    memcpy(ctx->ctab + ctx->ctab_len, bytes, len);
+    ctx->ctab_len += len;
+    return retval;
+} // add_ctab_bytes
+
+
+static inline uint32 add_ctab_string(Context *ctx, const char *str)
+{
+    return add_ctab_bytes(ctx, (const uint8 *) str, strlen(str) + 1);
+} // add_ctab_string
 
 
-FUCK THIS CODE.
-static void output_ctab(Context *ctx, const MOJOSHADER_symbol *symbols,
-                        unsigned int symbol_count)
+static uint32 add_ctab_typeinfo(Context *ctx, const MOJOSHADER_symbolTypeInfo *info);
+
+static uint32 add_ctab_members(Context *ctx, const MOJOSHADER_symbolTypeInfo *info)
 {
-    if (symbol_count == 0)
-        return;
-
     unsigned int i;
-    uint8 *bytes = NULL;
-    const char *creator = "MojoShader revision " MOJOSHADER_CHANGESET;
-    const size_t creatorlen = strlen(creator) + 1;
-    add_ctab_bytes(ctx, &bytes, &offset, creator, creatorlen);
-    add_ctab_bytes(ctx, &bytes, &offset, "", 1);  // !!! FIXME: target
+    const size_t len = info->member_count * CMEMBERINFO_SIZE;
+    uint8 *bytes = (uint8 *) Malloc(ctx, len);
+    if (bytes == NULL)
+        return 0;
+
+    union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr;
+    ptr.ui8 = bytes;
+    for (i = 0; i < info->member_count; i++)
+    {
+        const MOJOSHADER_symbolStructMember *member = &info->members[i];
+        *(ptr.ui32++) = SWAP32(add_ctab_string(ctx, member->name));
+        *(ptr.ui32++) = SWAP32(add_ctab_typeinfo(ctx, &member->info));
+    } // for
+
+    const uint32 retval = add_ctab_bytes(ctx, bytes, len);
+    Free(ctx, bytes);
+    return retval;
+} // add_ctab_members
+
 
-    // build all the unique D3DXSHADER_TYPEINFO structs into the bytes.
-    CTabTypeInfo *tinfo;
-    tinfo = (CTabTypeInfo *) Malloc(sizeof (CTabTypeInfo) * symbol_count);
-    if (tinfo == NULL)
-    {
-        Free(ctx, bytes);
-        return;
-    } // if
+static uint32 add_ctab_typeinfo(Context *ctx, const MOJOSHADER_symbolTypeInfo *info)
+{
+    uint8 bytes[CTYPEINFO_SIZE];
+    union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr;
+    ptr.ui8 = bytes;
+
+    *(ptr.ui16++) = SWAP16((uint16) info->parameter_class);
+    *(ptr.ui16++) = SWAP16((uint16) info->parameter_type);
+    *(ptr.ui16++) = SWAP16((uint16) info->rows);
+    *(ptr.ui16++) = SWAP16((uint16) info->columns);
+    *(ptr.ui16++) = SWAP16((uint16) info->elements);
+    *(ptr.ui16++) = SWAP16((uint16) info->member_count);
+    *(ptr.ui32++) = SWAP32(add_ctab_members(ctx, info));
 
+    return add_ctab_bytes(ctx, bytes, sizeof (bytes));
+} // add_ctab_typeinfo
+
+
+static uint32 add_ctab_info(Context *ctx, const MOJOSHADER_symbol *symbols,
+                            const unsigned int symbol_count)
+{
+    unsigned int i;
+    const size_t len = symbol_count * CINFO_SIZE;
+    uint8 *bytes = (uint8 *) Malloc(ctx, len);
+    if (bytes == NULL)
+        return 0;
+
+    union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr;
+    ptr.ui8 = bytes;
     for (i = 0; i < symbol_count; i++)
     {
-        // !!! FIXME: struct packing!
-        tinfo[i].parameter_class = SWAP16(symbols[i].parameter_class);
-        tinfo[i].parameter_type = SWAP16(symbols[i].parameter_type);
-        tinfo[i].rows = SWAP16(symbols[i].rows);
-        tinfo[i].columns = SWAP16(symbols[i].columns);
-        tinfo[i].elements = SWAP16(symbols[i].elements);
-        tinfo[i].structMembers = SWAP16(symbols[i].structMembers);
-        tinfo[i].structMembersInfo = SWAP32(0);  // !!! FIXME: points to DWORD name, DWORD typeinfo
-        add_ctab_bytes(ctx, &bytes, &offset, &tinfo[i], sizeof (tinfo[i]));
+        const MOJOSHADER_symbol *sym = &symbols[i];
+        *(ptr.ui32++) = SWAP32(add_ctab_string(ctx, sym->name));
+        *(ptr.ui16++) = SWAP16((uint16) sym->register_set);
+        *(ptr.ui16++) = SWAP16((uint16) sym->register_index);
+        *(ptr.ui16++) = SWAP16((uint16) sym->register_count);
+        *(ptr.ui16++) = SWAP16(0);  // reserved
+        *(ptr.ui32++) = SWAP32(add_ctab_typeinfo(ctx, &sym->info));
+        *(ptr.ui32++) = SWAP32(0);  // !!! FIXME: default value.
     } // for
 
+    const uint32 retval = add_ctab_bytes(ctx, bytes, len);
+    Free(ctx, bytes);
+    return retval;
+} // add_ctab_info
 
-    uint8 ctab = sdfkjsldkfjsdlkf;
-    uint32 *table = (uint32 *) ctab;
-    uint32 offset = CTAB_SIZE + (CINFO_SIZE * symbol_count);
-    uint8 *data = ctab + offset + sizeof (uint32);
-    union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } info;
-    info.ui8 = ctab + CTAB_SIZE + sizeof (uint32);
+
+static void output_ctab(Context *ctx, const MOJOSHADER_symbol *symbols,
+                        unsigned int symbol_count, const char *creator)
+{
+    ctx->ctab_len = CTAB_SIZE + sizeof (uint32);
 
+    uint8 bytes[CTAB_SIZE + sizeof (uint32)];
+    uint32 *table = (uint32 *) bytes;
     *(table++) = SWAP32(CTAB_ID);
-    *(table++) = SWAP32(28);
-    *(table++) = SWAP32(find_ctab_bytes(ctx, bytes, creator, creatorlen));
+    *(table++) = SWAP32(CTAB_SIZE);
+    *(table++) = SWAP32(add_ctab_string(ctx, creator));
     *(table++) = SWAP32(ctx->version_token);
     *(table++) = SWAP32(((uint32) symbol_count));
-    *(table++) = SWAP32(CTAB_SIZE);  // info array right after table.
-    *(table++) = SWAP32(0);
-    *(table++) = SWAP32(find_ctab_bytes(ctx, bytes, "", 1));  // !!! FIXME: target?
+    *(table++) = SWAP32(add_ctab_info(ctx, symbols, symbol_count));
+    *(table++) = SWAP32(0);  // build flags.
+    *(table++) = SWAP32(add_ctab_string(ctx, ""));  // !!! FIXME: target?
+    memcpy(ctx->ctab, bytes, sizeof (bytes));
+    output_comment_bytes(ctx, ctx->ctab, ctx->ctab_len);
 
-    assert( ((uint8 *) table) == info.ui8 );
-    for (i = 0; i < symbol_count; i++)
-    {
-        const char *name = symbols[i].name;
-        const size_t namelen = strlen(symbols[i].name) + 1;
-        add_ctab_bytes(bytes, &offset, name, namelen);
-        *(info.ui32++) = SWAP32(add_ctab_string(ctx, &data, &offset, symbols[i].name));
-        *(info.ui16++) = SWAP16((uint16) symbols[i].register_set);
-        *(info.ui16++) = SWAP16((uint16) symbols[i].register_index);
-        *(info.ui16++) = SWAP16((uint16) symbols[i].register_count);
-        *(info.ui16++) = SWAP16(0);  // reserved
-
-    MOJOSHADER_symbolClass parameter_class;
-    MOJOSHADER_symbolType parameter_type;
-    unsigned int rows;
-    unsigned int columns;
-    unsigned int elements;
-    unsigned int structMembers;
-    unsigned int bytes;
-    void *default_value;
-
-    } // for
-
-    assert( info.ui8 == origdata);
-
-    output_comment_bytes(ctx, ctab, (size_t) (ptr - ctab));
+    Free(ctx, ctx->ctab);
+    ctx->ctab = NULL;
+    ctx->ctab_len = 0;
+    ctx->ctab_allocation = 0;
 } // output_ctab
 
 
@@ -1784,7 +1838,11 @@
     // make error messages sane if CTAB fails, etc.
     ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED;
 
-    output_ctab(ctx, symbols, symbol_count);
+    const char *creator = "MojoShader revision " MOJOSHADER_CHANGESET;
+    if (symbol_count > 0)
+        output_ctab(ctx, symbols, symbol_count, creator);
+    else
+        output_comment_string(ctx, creator);
 
     int i;
     for (i = 0; i < comment_count; i++)
@@ -1798,6 +1856,9 @@
 // API entry point...
 
 const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *source,
+                            const char **comments, unsigned int comment_count,
+                            const MOJOSHADER_symbol *symbols,
+                            unsigned int symbol_count,
                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
 {
     MOJOSHADER_parseData *retval = NULL;
@@ -1813,14 +1874,7 @@
     // Version token always comes first.
     ctx->parse_phase = MOJOSHADER_PARSEPHASE_WORKING;
     parse_version_token(ctx);
-
-    ctx->started_parsing = 0;  // make error messages sane if CTAB fails, etc.
-    const char *credit = "Generated by MojoShader assembler revision "
-                         MOJOSHADER_CHANGESET
-                         ", http://icculus.org/mojoshader/";
-    output_comment_string(ctx, credit);
-
-    // !!! FIXME: insert CTAB here.
+    output_comments(ctx, comments, comment_count, symbols, symbol_count);
 
     // parse out the rest of the tokens after the version token...
     while (nexttoken(ctx, 1, 1, 0, 1) == NOFAIL)
--- a/mojoshader_internal.h	Fri Dec 19 00:05:20 2008 -0500
+++ b/mojoshader_internal.h	Sat Dec 20 05:48:38 2008 -0500
@@ -117,6 +117,12 @@
 #   define SWAP32(x) (x)
 #endif
 
+// This is the ID for a D3DXSHADER_CONSTANTTABLE in the bytecode comments.
+#define CTAB_ID 0x42415443  // 0x42415443 == 'CTAB'
+#define CTAB_SIZE 28  // sizeof (D3DXSHADER_CONSTANTTABLE).
+#define CINFO_SIZE 20  // sizeof (D3DXSHADER_CONSTANTINFO).
+#define CTYPEINFO_SIZE 16  // sizeof (D3DXSHADER_TYPEINFO).
+#define CMEMBERINFO_SIZE 8  // sizeof (D3DXSHADER_STRUCTMEMBERINFO)
 
 // we need to reference these by explicit value occasionally...
 #define OPCODE_RET 28
@@ -138,7 +144,6 @@
 static void internal_free(void *ptr, void *d) { free(ptr); }
 #endif
 
-
 // result modifiers.
 // !!! FIXME: why isn't this an enum?
 #define MOD_SATURATE 0x01