From 6c918ef5696034d55848b3dab1bfdad4634da6b4 Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Sat, 20 Dec 2008 05:48:38 -0500 Subject: [PATCH] First (incomplete!) shot at more robust CTAB support. Assembler can now add it, but we still need to parse it better and report it all back to the app in MOJOSHADER_parse(). --- assemble.c | 2 +- finderrors.c | 2 +- mojoshader.c | 23 +++-- mojoshader.h | 111 ++++++++++++++++++++- mojoshader_assembler.c | 220 +++++++++++++++++++++++++---------------- mojoshader_internal.h | 7 +- 6 files changed, 272 insertions(+), 93 deletions(-) diff --git a/assemble.c b/assemble.c index 19f4a690..22f90ecc 100644 --- a/assemble.c +++ b/assemble.c @@ -23,7 +23,7 @@ static int assemble(const char *buf, const char *outfile) const MOJOSHADER_parseData *pd; int retval = 0; - pd = MOJOSHADER_assemble(buf, NULL, NULL, NULL); + pd = MOJOSHADER_assemble(buf, NULL, 0, NULL, 0, NULL, NULL, NULL); if (pd->error != NULL) printf("ERROR: (line %d) %s\n", pd->error_position, pd->error); else diff --git a/finderrors.c b/finderrors.c index 00925620..30fd17ea 100644 --- a/finderrors.c +++ b/finderrors.c @@ -84,7 +84,7 @@ static int do_file(const char *profile, const char *dname, const char *fn, int * const MOJOSHADER_parseData *a; buf[rc] = '\0'; // make sure the source is null-terminated. - a = MOJOSHADER_assemble((char *) buf, 0, 0, 0); + a = MOJOSHADER_assemble((char *) buf, 0, 0, 0, 0, 0, 0, 0); if (a->error) { report("FAIL: %s (line %d) %s\n", fname, a->error_position, a->error); diff --git a/mojoshader.c b/mojoshader.c index 6064429c..e1edb6c2 100644 --- a/mojoshader.c +++ b/mojoshader.c @@ -5737,6 +5737,7 @@ static ConstantsList *alloc_constant_listitem(Context *ctx) return item; } // alloc_constant_listitem + static void state_DEF(Context *ctx) { const RegisterType regtype = ctx->dest_arg.regtype; @@ -6517,24 +6518,22 @@ static void parse_constant_table(Context *ctx, const uint32 bytes) const uint32 target = SWAP32(ctx->tokens[8]); uint32 i = 0; - if (id != 0x42415443) // 0x42415443 == 'CTAB' + if (id != CTAB_ID) return; // not the constant table. - if (size != 28) + if (size != CTAB_SIZE) return; // only handle this version of the struct. if (version != ctx->version_token) goto corrupt_ctab; if (creator >= bytes) goto corrupt_ctab; - if ((constantinfo + (constants * 20)) >= bytes) goto corrupt_ctab; + if ((constantinfo + (constants * CINFO_SIZE)) >= bytes) goto corrupt_ctab; if (target >= bytes) goto corrupt_ctab; ctx->have_ctab = 1; for (i = 0; i < constants; i++) { - // we only care about deciding which variables might be arrays at - // the moment, but there's lots of other good info in the CTAB. - const uint8 *ptr = start + constantinfo + (i * 20); + const uint8 *ptr = start + constantinfo + (i * CINFO_SIZE); const uint32 name = SWAP32(*((uint32 *) (ptr + 0))); const uint16 regset = SWAP16(*((uint16 *) (ptr + 4))); const uint16 regidx = SWAP16(*((uint16 *) (ptr + 6))); @@ -7471,6 +7470,18 @@ void MOJOSHADER_freeParseData(const MOJOSHADER_parseData *_data) f((void *) data->samplers, d); } // if + if (data->symbols != NULL) + { + for (i = 0; i < data->symbol_count; i++) + { + if (data->symbols[i].name != NULL) + f((void *) data->symbols[i].name, d); + if (data->symbols[i].default_value != NULL) + f((void *) data->symbols[i].default_value, d); + } // for + f((void *) data->symbols, d); + } // if + if ((data->error != NULL) && (data->error != out_of_mem_str)) f((void *) data->error, d); diff --git a/mojoshader.h b/mojoshader.h index 916b7720..bce76ed1 100644 --- a/mojoshader.h +++ b/mojoshader.h @@ -239,9 +239,90 @@ typedef struct MOJOSHADER_swizzle } MOJOSHADER_swizzle; +/* + * MOJOSHADER_symbol data. + * + * These are used to expose high-level information in shader bytecode. + * They associate HLSL variables with registers. This data is used for both + * debugging and optimization. + */ + +typedef enum +{ + MOJOSHADER_SYMREGSET_BOOL, + MOJOSHADER_SYMREGSET_INT4, + MOJOSHADER_SYMREGSET_FLOAT4, + MOJOSHADER_SYMREGSET_SAMPLER, +} MOJOSHADER_symbolRegisterSet; + +typedef enum +{ + MOJOSHADER_SYMCLASS_SCALAR, + MOJOSHADER_SYMCLASS_VECTOR, + MOJOSHADER_SYMCLASS_MATRIX_ROWS, + MOJOSHADER_SYMCLASS_MATRIX_COLUMNS, + MOJOSHADER_SYMCLASS_OBJECT, + MOJOSHADER_SYMCLASS_STRUCT, +} MOJOSHADER_symbolClass; + +typedef enum +{ + MOJOSHADER_SYMTYPE_VOID, + MOJOSHADER_SYMTYPE_BOOL, + MOJOSHADER_SYMTYPE_INT, + MOJOSHADER_SYMTYPE_FLOAT, + MOJOSHADER_SYMTYPE_STRING, + MOJOSHADER_SYMTYPE_TEXTURE, + MOJOSHADER_SYMTYPE_TEXTURE1D, + MOJOSHADER_SYMTYPE_TEXTURE2D, + MOJOSHADER_SYMTYPE_TEXTURE3D, + MOJOSHADER_SYMTYPE_TEXTURECUBE, + MOJOSHADER_SYMTYPE_SAMPLER, + MOJOSHADER_SYMTYPE_SAMPLER1D, + MOJOSHADER_SYMTYPE_SAMPLER2D, + MOJOSHADER_SYMTYPE_SAMPLER3D, + MOJOSHADER_SYMTYPE_SAMPLERCUBE, + MOJOSHADER_SYMTYPE_PIXELSHADER, + MOJOSHADER_SYMTYPE_VERTEXSHADER, + MOJOSHADER_SYMTYPE_PIXELFRAGMENT, + MOJOSHADER_SYMTYPE_VERTEXFRAGMENT, + MOJOSHADER_SYMTYPE_UNSUPPORTED, +} MOJOSHADER_symbolType; + +typedef struct MOJOSHADER_symbolStructMember MOJOSHADER_symbolStructMember; + +typedef struct MOJOSHADER_symbolTypeInfo +{ + MOJOSHADER_symbolClass parameter_class; + MOJOSHADER_symbolType parameter_type; + unsigned int rows; + unsigned int columns; + unsigned int elements; + unsigned int member_count; + MOJOSHADER_symbolStructMember *members; +} MOJOSHADER_symbolTypeInfo; + +struct MOJOSHADER_symbolStructMember +{ + const char *name; + MOJOSHADER_symbolTypeInfo info; +}; + +typedef struct MOJOSHADER_symbol +{ + const char *name; + MOJOSHADER_symbolRegisterSet register_set; + unsigned int register_index; + unsigned int register_count; + MOJOSHADER_symbolTypeInfo info; + void *default_value; +} MOJOSHADER_symbol; + + /* * Structure used to return data from parsing of a shader... */ +/* !!! FIXME: most of these ints should be unsigned. */ typedef struct MOJOSHADER_parseData { /* @@ -372,6 +453,20 @@ typedef struct MOJOSHADER_parseData */ MOJOSHADER_swizzle *swizzles; + /* + * The number of elements pointed to by (symbols). + */ + int symbol_count; + + /* + * (symbol_count) elements of data that specify high-level symbol data + * for the shader. This will be parsed from the CTAB section + * in bytecode, and will be a copy of what you provide to + * MOJOSHADER_assemble(). This data is optional. + * This can be NULL on error or if (symbol_count) is zero. + */ + MOJOSHADER_symbol *symbols; + /* * This is the malloc implementation you passed to MOJOSHADER_parse(). */ @@ -504,6 +599,17 @@ void MOJOSHADER_freeParseData(const MOJOSHADER_parseData *data); * (source) is an ASCII, NULL-terminated string of valid Direct3D shader * assembly source code. * + * (comments) points to (comment_count) NULL-terminated ASCII strings, and + * can be NULL. These strings are inserted as comments in the bytecode. + * + * (symbols) points to (symbol_count) symbol structs, and can be NULL. These + * become a CTAB field in the bytecode. This is optional, but + * MOJOSHADER_parse() needs CTAB data for all arrays used in a program, or + * relative addressing will not be permitted, so you'll want to at least + * provide symbol information for those. The symbol data is 100% trusted + * at this time; it will not be checked to see if it matches what was + * assembled in any way whatsoever. + * * This will return a MOJOSHADER_parseData(), like MOJOSHADER_parse() would, * except the profile will be MOJOSHADER_PROFILE_BYTECODE and the output * will be the assembled bytecode instead of some other language. This output @@ -526,7 +632,10 @@ void MOJOSHADER_freeParseData(const MOJOSHADER_parseData *data); * to assemble several shaders on separate CPU cores at the same time. */ const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *source, - MOJOSHADER_malloc m, MOJOSHADER_free f, void *d); + const char **comments, unsigned int comment_count, + const MOJOSHADER_symbol *symbols, + unsigned int symbol_count, + MOJOSHADER_malloc m, MOJOSHADER_free f, void *d); diff --git a/mojoshader_assembler.c b/mojoshader_assembler.c index 86e244ca..6429315b 100644 --- a/mojoshader_assembler.c +++ b/mojoshader_assembler.c @@ -39,11 +39,15 @@ struct Context MOJOSHADER_shaderType shader_type; uint8 major_ver; uint8 minor_ver; + uint32 version_token; uint32 tokenbuf[16]; int tokenbufpos; DestArgInfo dest_arg; uint32 *output; uint32 *token_to_line; + uint8 *ctab; + uint32 ctab_len; + uint32 ctab_allocation; size_t output_len; size_t output_allocation; }; @@ -1568,7 +1572,8 @@ static int parse_version_token(Context *ctx) if (require_endline(ctx) == FAIL) return FAIL; - output_token(ctx, (shader_type << 16) | (major << 8) | (minor << 0) ); + ctx->version_token = (shader_type << 16) | (major << 8) | (minor << 0); + output_token(ctx, ctx->version_token); return NOFAIL; } // parse_version_token @@ -1640,6 +1645,8 @@ static void destroy_context(Context *ctx) f(ctx->output, d); if (ctx->token_to_line != NULL) f(ctx->token_to_line, d); + if (ctx->ctab != NULL) + f(ctx->ctab, d); f(ctx, d); } // if } // destroy_context @@ -1679,97 +1686,144 @@ static const MOJOSHADER_parseData *build_failed_assembly(Context *ctx) } // build_failed_assembly -typedef struct CTabTypeInfo +static uint32 add_ctab_bytes(Context *ctx, const uint8 *bytes, const size_t len) { - uint16 parameter_class; - uint16 parameter_type; - uint16 rows; - uint16 columns; - uint16 elements; - uint16 structMembers; - uint32 structMemberInfo; -} CTabTypeInfo; + const size_t extra = CTAB_SIZE + sizeof (uint32); + if (len <= (ctx->ctab_len - extra)) + { + void *ptr = ctx->ctab + extra; + while ((ptr = memchr(ptr, bytes[0], ctx->ctab_len - len)) != NULL) + { + if (memcmp(ptr, bytes, len) == 0) // already have it? + return ( (uint32) (ctx->ctab - ((uint8 *) ptr)) ); + ptr++; + } // while + } // if + // add it to the byte pile... -FUCK THIS CODE. -static void output_ctab(Context *ctx, const MOJOSHADER_symbol *symbols, - unsigned int symbol_count) + // verify allocation. + const size_t newsize = (ctx->ctab_len + len); + if (ctx->ctab_allocation < newsize) + { + const size_t additional = 4 * 1024; + while (ctx->ctab_allocation < newsize) + ctx->ctab_allocation += additional; + void *ptr = Malloc(ctx, ctx->ctab_allocation); + if (ptr == NULL) + return 0; + memcpy(ptr, ctx->ctab, ctx->ctab_len); + Free(ctx, ctx->ctab); + ctx->ctab = (uint8 *) ptr; + } // if + + const uint32 retval = ctx->ctab_len; + memcpy(ctx->ctab + ctx->ctab_len, bytes, len); + ctx->ctab_len += len; + return retval; +} // add_ctab_bytes + + +static inline uint32 add_ctab_string(Context *ctx, const char *str) { - if (symbol_count == 0) - return; + return add_ctab_bytes(ctx, (const uint8 *) str, strlen(str) + 1); +} // add_ctab_string + + +static uint32 add_ctab_typeinfo(Context *ctx, const MOJOSHADER_symbolTypeInfo *info); +static uint32 add_ctab_members(Context *ctx, const MOJOSHADER_symbolTypeInfo *info) +{ unsigned int i; - uint8 *bytes = NULL; - const char *creator = "MojoShader revision " MOJOSHADER_CHANGESET; - const size_t creatorlen = strlen(creator) + 1; - add_ctab_bytes(ctx, &bytes, &offset, creator, creatorlen); - add_ctab_bytes(ctx, &bytes, &offset, "", 1); // !!! FIXME: target - - // build all the unique D3DXSHADER_TYPEINFO structs into the bytes. - CTabTypeInfo *tinfo; - tinfo = (CTabTypeInfo *) Malloc(sizeof (CTabTypeInfo) * symbol_count); - if (tinfo == NULL) - { - Free(ctx, bytes); - return; - } // if + const size_t len = info->member_count * CMEMBERINFO_SIZE; + uint8 *bytes = (uint8 *) Malloc(ctx, len); + if (bytes == NULL) + return 0; - for (i = 0; i < symbol_count; i++) + union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr; + ptr.ui8 = bytes; + for (i = 0; i < info->member_count; i++) { - // !!! FIXME: struct packing! - tinfo[i].parameter_class = SWAP16(symbols[i].parameter_class); - tinfo[i].parameter_type = SWAP16(symbols[i].parameter_type); - tinfo[i].rows = SWAP16(symbols[i].rows); - tinfo[i].columns = SWAP16(symbols[i].columns); - tinfo[i].elements = SWAP16(symbols[i].elements); - tinfo[i].structMembers = SWAP16(symbols[i].structMembers); - tinfo[i].structMembersInfo = SWAP32(0); // !!! FIXME: points to DWORD name, DWORD typeinfo - add_ctab_bytes(ctx, &bytes, &offset, &tinfo[i], sizeof (tinfo[i])); + const MOJOSHADER_symbolStructMember *member = &info->members[i]; + *(ptr.ui32++) = SWAP32(add_ctab_string(ctx, member->name)); + *(ptr.ui32++) = SWAP32(add_ctab_typeinfo(ctx, &member->info)); } // for + const uint32 retval = add_ctab_bytes(ctx, bytes, len); + Free(ctx, bytes); + return retval; +} // add_ctab_members - uint8 ctab = sdfkjsldkfjsdlkf; - uint32 *table = (uint32 *) ctab; - uint32 offset = CTAB_SIZE + (CINFO_SIZE * symbol_count); - uint8 *data = ctab + offset + sizeof (uint32); - union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } info; - info.ui8 = ctab + CTAB_SIZE + sizeof (uint32); - *(table++) = SWAP32(CTAB_ID); - *(table++) = SWAP32(28); - *(table++) = SWAP32(find_ctab_bytes(ctx, bytes, creator, creatorlen)); - *(table++) = SWAP32(ctx->version_token); - *(table++) = SWAP32(((uint32) symbol_count)); - *(table++) = SWAP32(CTAB_SIZE); // info array right after table. - *(table++) = SWAP32(0); - *(table++) = SWAP32(find_ctab_bytes(ctx, bytes, "", 1)); // !!! FIXME: target? +static uint32 add_ctab_typeinfo(Context *ctx, const MOJOSHADER_symbolTypeInfo *info) +{ + uint8 bytes[CTYPEINFO_SIZE]; + union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr; + ptr.ui8 = bytes; + + *(ptr.ui16++) = SWAP16((uint16) info->parameter_class); + *(ptr.ui16++) = SWAP16((uint16) info->parameter_type); + *(ptr.ui16++) = SWAP16((uint16) info->rows); + *(ptr.ui16++) = SWAP16((uint16) info->columns); + *(ptr.ui16++) = SWAP16((uint16) info->elements); + *(ptr.ui16++) = SWAP16((uint16) info->member_count); + *(ptr.ui32++) = SWAP32(add_ctab_members(ctx, info)); + + return add_ctab_bytes(ctx, bytes, sizeof (bytes)); +} // add_ctab_typeinfo + - assert( ((uint8 *) table) == info.ui8 ); +static uint32 add_ctab_info(Context *ctx, const MOJOSHADER_symbol *symbols, + const unsigned int symbol_count) +{ + unsigned int i; + const size_t len = symbol_count * CINFO_SIZE; + uint8 *bytes = (uint8 *) Malloc(ctx, len); + if (bytes == NULL) + return 0; + + union { uint8 *ui8; uint16 *ui16; uint32 *ui32; } ptr; + ptr.ui8 = bytes; for (i = 0; i < symbol_count; i++) { - const char *name = symbols[i].name; - const size_t namelen = strlen(symbols[i].name) + 1; - add_ctab_bytes(bytes, &offset, name, namelen); - *(info.ui32++) = SWAP32(add_ctab_string(ctx, &data, &offset, symbols[i].name)); - *(info.ui16++) = SWAP16((uint16) symbols[i].register_set); - *(info.ui16++) = SWAP16((uint16) symbols[i].register_index); - *(info.ui16++) = SWAP16((uint16) symbols[i].register_count); - *(info.ui16++) = SWAP16(0); // reserved - - MOJOSHADER_symbolClass parameter_class; - MOJOSHADER_symbolType parameter_type; - unsigned int rows; - unsigned int columns; - unsigned int elements; - unsigned int structMembers; - unsigned int bytes; - void *default_value; - + const MOJOSHADER_symbol *sym = &symbols[i]; + *(ptr.ui32++) = SWAP32(add_ctab_string(ctx, sym->name)); + *(ptr.ui16++) = SWAP16((uint16) sym->register_set); + *(ptr.ui16++) = SWAP16((uint16) sym->register_index); + *(ptr.ui16++) = SWAP16((uint16) sym->register_count); + *(ptr.ui16++) = SWAP16(0); // reserved + *(ptr.ui32++) = SWAP32(add_ctab_typeinfo(ctx, &sym->info)); + *(ptr.ui32++) = SWAP32(0); // !!! FIXME: default value. } // for - assert( info.ui8 == origdata); + const uint32 retval = add_ctab_bytes(ctx, bytes, len); + Free(ctx, bytes); + return retval; +} // add_ctab_info + + +static void output_ctab(Context *ctx, const MOJOSHADER_symbol *symbols, + unsigned int symbol_count, const char *creator) +{ + ctx->ctab_len = CTAB_SIZE + sizeof (uint32); - output_comment_bytes(ctx, ctab, (size_t) (ptr - ctab)); + uint8 bytes[CTAB_SIZE + sizeof (uint32)]; + uint32 *table = (uint32 *) bytes; + *(table++) = SWAP32(CTAB_ID); + *(table++) = SWAP32(CTAB_SIZE); + *(table++) = SWAP32(add_ctab_string(ctx, creator)); + *(table++) = SWAP32(ctx->version_token); + *(table++) = SWAP32(((uint32) symbol_count)); + *(table++) = SWAP32(add_ctab_info(ctx, symbols, symbol_count)); + *(table++) = SWAP32(0); // build flags. + *(table++) = SWAP32(add_ctab_string(ctx, "")); // !!! FIXME: target? + memcpy(ctx->ctab, bytes, sizeof (bytes)); + output_comment_bytes(ctx, ctx->ctab, ctx->ctab_len); + + Free(ctx, ctx->ctab); + ctx->ctab = NULL; + ctx->ctab_len = 0; + ctx->ctab_allocation = 0; } // output_ctab @@ -1784,7 +1838,11 @@ static void output_comments(Context *ctx, const char **comments, // make error messages sane if CTAB fails, etc. ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED; - output_ctab(ctx, symbols, symbol_count); + const char *creator = "MojoShader revision " MOJOSHADER_CHANGESET; + if (symbol_count > 0) + output_ctab(ctx, symbols, symbol_count, creator); + else + output_comment_string(ctx, creator); int i; for (i = 0; i < comment_count; i++) @@ -1798,6 +1856,9 @@ static void output_comments(Context *ctx, const char **comments, // API entry point... const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *source, + const char **comments, unsigned int comment_count, + const MOJOSHADER_symbol *symbols, + unsigned int symbol_count, MOJOSHADER_malloc m, MOJOSHADER_free f, void *d) { MOJOSHADER_parseData *retval = NULL; @@ -1813,14 +1874,7 @@ const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *source, // Version token always comes first. ctx->parse_phase = MOJOSHADER_PARSEPHASE_WORKING; parse_version_token(ctx); - - ctx->started_parsing = 0; // make error messages sane if CTAB fails, etc. - const char *credit = "Generated by MojoShader assembler revision " - MOJOSHADER_CHANGESET - ", http://icculus.org/mojoshader/"; - output_comment_string(ctx, credit); - - // !!! FIXME: insert CTAB here. + output_comments(ctx, comments, comment_count, symbols, symbol_count); // parse out the rest of the tokens after the version token... while (nexttoken(ctx, 1, 1, 0, 1) == NOFAIL) diff --git a/mojoshader_internal.h b/mojoshader_internal.h index 4de0c367..302a7b64 100644 --- a/mojoshader_internal.h +++ b/mojoshader_internal.h @@ -117,6 +117,12 @@ typedef int32_t int32; # define SWAP32(x) (x) #endif +// This is the ID for a D3DXSHADER_CONSTANTTABLE in the bytecode comments. +#define CTAB_ID 0x42415443 // 0x42415443 == 'CTAB' +#define CTAB_SIZE 28 // sizeof (D3DXSHADER_CONSTANTTABLE). +#define CINFO_SIZE 20 // sizeof (D3DXSHADER_CONSTANTINFO). +#define CTYPEINFO_SIZE 16 // sizeof (D3DXSHADER_TYPEINFO). +#define CMEMBERINFO_SIZE 8 // sizeof (D3DXSHADER_STRUCTMEMBERINFO) // we need to reference these by explicit value occasionally... #define OPCODE_RET 28 @@ -138,7 +144,6 @@ static void *internal_malloc(int bytes, void *d) { return malloc(bytes); } static void internal_free(void *ptr, void *d) { free(ptr); } #endif - // result modifiers. // !!! FIXME: why isn't this an enum? #define MOD_SATURATE 0x01