From 6f7d26b9e9c3bc8b7336ef57d1d9abb857bd3436 Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Sun, 22 May 2011 03:32:10 -0400 Subject: [PATCH] Initial work on parsing binary Effects files. --- CMakeLists.txt | 1 + mojoshader.h | 124 ++++++++++++++ mojoshader_effects.c | 374 +++++++++++++++++++++++++++++++++++++++++++ utils/testparse.c | 145 ++++++++++++++--- 4 files changed, 624 insertions(+), 20 deletions(-) create mode 100644 mojoshader_effects.c diff --git a/CMakeLists.txt b/CMakeLists.txt index bbc96151..23dfe60d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ ADD_CUSTOM_COMMAND( ADD_LIBRARY(mojoshader STATIC mojoshader.c mojoshader_common.c + mojoshader_effects.c mojoshader_compiler.c mojoshader_preprocessor.c mojoshader_lexer.c diff --git a/mojoshader.h b/mojoshader.h index a0e9f820..c7391a9b 100644 --- a/mojoshader.h +++ b/mojoshader.h @@ -619,6 +619,130 @@ const MOJOSHADER_parseData *MOJOSHADER_parse(const char *profile, void MOJOSHADER_freeParseData(const MOJOSHADER_parseData *data); +/* Effects interface... */ /* !!! FIXME: THIS API IS NOT STABLE YET! */ + +typedef struct MOJOSHADER_effectState +{ + unsigned int type; +} MOJOSHADER_effectState; + +typedef struct MOJOSHADER_effectPass +{ + const char *name; + unsigned int state_count; + MOJOSHADER_effectState *states; +} MOJOSHADER_effectPass; + +typedef struct MOJOSHADER_effectTechnique +{ + const char *name; + unsigned int pass_count; + MOJOSHADER_effectPass *passes; +} MOJOSHADER_effectTechnique; + +typedef struct MOJOSHADER_effectTexture +{ + unsigned int param; + const char *name; +} MOJOSHADER_effectTexture; + +typedef struct MOJOSHADER_effectShader +{ + unsigned int technique; + unsigned int pass; + const MOJOSHADER_parseData *shader; +} MOJOSHADER_effectShader; + +/* + * Structure used to return data from parsing of an effect file... + */ +/* !!! FIXME: most of these ints should be unsigned. */ +typedef struct MOJOSHADER_effect +{ + /* + * The number of elements pointed to by (errors). + */ + int error_count; + + /* + * (error_count) elements of data that specify errors that were generated + * by parsing this shader. + * This can be NULL if there were no errors or if (error_count) is zero. + */ + MOJOSHADER_error *errors; + + /* + * The name of the profile used to parse the shader. Will be NULL on error. + */ + const char *profile; + + /* + * The number of elements pointed to by (techniques). + */ + int technique_count; + + /* + * (technique_count) elements of data that specify techniques used in + * this effect. Each technique contains a series of passes, and each pass + * specifies state and shaders that affect rendering. + * This can be NULL on error or if (technique_count) is zero. + */ + MOJOSHADER_effectTechnique *techniques; + + /* + * The number of elements pointed to by (textures). + */ + int texture_count; + + /* + * (texture_count) elements of data that specify textures used in + * this effect. + * This can be NULL on error or if (texture_count) is zero. + */ + MOJOSHADER_effectTexture *textures; + + /* + * The number of elements pointed to by (shaders). + */ + int shader_count; + + /* + * (shader_count) elements of data that specify shaders used in + * this effect. + * This can be NULL on error or if (shader_count) is zero. + */ + MOJOSHADER_effectShader *shaders; + + /* + * This is the malloc implementation you passed to MOJOSHADER_parseEffect(). + */ + MOJOSHADER_malloc malloc; + + /* + * This is the free implementation you passed to MOJOSHADER_parseEffect(). + */ + MOJOSHADER_free free; + + /* + * This is the pointer you passed as opaque data for your allocator. + */ + void *malloc_data; +} MOJOSHADER_effect; + +/* !!! FIXME: document me. */ +const MOJOSHADER_effect *MOJOSHADER_parseEffect(const char *profile, + const unsigned char *buf, + const unsigned int _len, + const MOJOSHADER_swizzle *swiz, + const unsigned int swizcount, + MOJOSHADER_malloc m, + MOJOSHADER_free f, + void *d); + + +/* !!! FIXME: document me. */ +void MOJOSHADER_freeEffect(const MOJOSHADER_effect *effect); + /* Preprocessor interface... */ diff --git a/mojoshader_effects.c b/mojoshader_effects.c new file mode 100644 index 00000000..2126b9bd --- /dev/null +++ b/mojoshader_effects.c @@ -0,0 +1,374 @@ +/** + * MojoShader; generate shader programs from bytecode of compiled + * Direct3D shaders. + * + * Please see the file LICENSE.txt in the source's root directory. + * + * This file written by Ryan C. Gordon. + */ + +#define __MOJOSHADER_INTERNAL__ 1 +#include "mojoshader_internal.h" + + +static MOJOSHADER_effect MOJOSHADER_out_of_mem_effect = { + 1, &MOJOSHADER_out_of_mem_error, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static uint32 readui32(const uint8 **_ptr, uint32 *_len) +{ + uint32 retval = 0; + if (*_len < sizeof (retval)) + *_len = 0; + else + { + const uint32 *ptr = (const uint32 *) *_ptr; + retval = SWAP32(*ptr); + *_ptr += sizeof (retval); + *_len -= sizeof (retval); + } // else + return retval; +} // readui32 + +// !!! FIXME: this is sort of a big, ugly function. +const MOJOSHADER_effect *MOJOSHADER_parseEffect(const char *profile, + const unsigned char *buf, + const unsigned int _len, + const MOJOSHADER_swizzle *swiz, + const unsigned int swizcount, + MOJOSHADER_malloc m, + MOJOSHADER_free f, + void *d) +{ + if ( ((m == NULL) && (f != NULL)) || ((m != NULL) && (f == NULL)) ) + return &MOJOSHADER_out_of_mem_effect; // supply both or neither. + + if (m == NULL) m = MOJOSHADER_internal_malloc; + if (f == NULL) f = MOJOSHADER_internal_free; + + MOJOSHADER_effect *retval = m(sizeof (MOJOSHADER_effect), d); + if (retval == NULL) + return &MOJOSHADER_out_of_mem_effect; // supply both or neither. + memset(retval, '\0', sizeof (*retval)); + + retval->malloc = m; + retval->free = f; + retval->malloc_data = d; + + const uint8 *ptr = (const uint8 *) buf; + uint32 len = (uint32) _len; + size_t siz = 0; + int i, j, k; + + if (len < 8) + goto parseEffect_unexpectedEOF; + + if (readui32(&ptr, &len) != 0xFEFF0901) // !!! FIXME: is this always magic? + goto parseEffect_notAnEffectsFile; + else + { + const uint32 offset = readui32(&ptr, &len); + if (offset > len) + goto parseEffect_unexpectedEOF; + ptr += offset; + len -= offset; + } // else + + // params... + + if (len < 16) + goto parseEffect_unexpectedEOF; + + const uint32 numparams = readui32(&ptr, &len); + const uint32 numtechniques = readui32(&ptr, &len); + + readui32(&ptr, &len); // !!! FIXME: there are 8 unknown bytes here. + readui32(&ptr, &len); + + for (i = 0; i < numparams; i++) + { + if (len < 16) + goto parseEffect_unexpectedEOF; + + /*const uint32 startoffset = */ readui32(&ptr, &len); + /*const uint32 endoffset = */ readui32(&ptr, &len); + readui32(&ptr, &len); // !!! FIXME: don't know what this is. + const uint32 numannos = readui32(&ptr, &len); + for (j = 0; j < numannos; j++) + { + if (len < 8) + goto parseEffect_unexpectedEOF; + // !!! FIXME: parse annotations. + readui32(&ptr, &len); + readui32(&ptr, &len); + } // for + } // for + + uint32 numshaders = 0; // we'll calculate this later. + + // techniques... + + if (numtechniques > 0) + { + siz = sizeof (MOJOSHADER_effectTechnique) * numtechniques; + retval->techniques = (MOJOSHADER_effectTechnique *) m(siz, d); + if (retval->techniques == NULL) + goto parseEffect_outOfMemory; + memset(retval->techniques, '\0', siz); + + retval->technique_count = numtechniques; + + for (i = 0; i < numtechniques; i++) + { + if (len < 12) + goto parseEffect_unexpectedEOF; + + MOJOSHADER_effectTechnique *technique = &retval->techniques[i]; + + // !!! FIXME: is this always 12? + const uint32 nameoffset = readui32(&ptr, &len) + 12; + readui32(&ptr, &len); // !!! FIXME: don't know what this field does. + const uint32 numpasses = readui32(&ptr, &len); + + if (nameoffset >= _len) + goto parseEffect_unexpectedEOF; + + if (numpasses > 0) + { + // !!! FIXME: verify this doesn't go past EOF looking for a null. + siz = strlen(((const char *) buf) + nameoffset) + 1; + technique->name = (char *) m(siz, d); + if (technique->name == NULL) + goto parseEffect_outOfMemory; + strcpy((char *) technique->name, ((const char *) buf) + nameoffset); + + technique->pass_count = numpasses; + + siz = sizeof (MOJOSHADER_effectPass) * numpasses; + technique->passes = (MOJOSHADER_effectPass *) m(siz, d); + if (technique->passes == NULL) + goto parseEffect_outOfMemory; + memset(technique->passes, '\0', siz); + + for (j = 0; j < numpasses; j++) + { + if (len < 12) + goto parseEffect_unexpectedEOF; + + MOJOSHADER_effectPass *pass = &technique->passes[j]; + + // !!! FIXME: is this always 12? + const uint32 passnameoffset = readui32(&ptr, &len) + 12; + readui32(&ptr, &len); // !!! FIXME: don't know what this field does. + const uint32 numstates = readui32(&ptr, &len); + + if (passnameoffset >= _len) + goto parseEffect_unexpectedEOF; + + // !!! FIXME: verify this doesn't go past EOF looking for a null. + siz = strlen(((const char *) buf) + passnameoffset) + 1; + pass->name = (char *) m(siz, d); + if (pass->name == NULL) + goto parseEffect_outOfMemory; + strcpy((char *) pass->name, ((const char *) buf) + passnameoffset); + + if (numstates > 0) + { + pass->state_count = numstates; + + siz = sizeof (MOJOSHADER_effectState) * numstates; + pass->states = (MOJOSHADER_effectState *) m(siz, d); + if (pass->states == NULL) + goto parseEffect_outOfMemory; + memset(pass->states, '\0', siz); + + for (k = 0; k < numstates; k++) + { + if (len < 16) + goto parseEffect_unexpectedEOF; + + MOJOSHADER_effectState *state = &pass->states[k]; + const uint32 type = readui32(&ptr, &len); + readui32(&ptr, &len); // !!! FIXME: don't know what this field does. + /*const uint32 offsetend = */ readui32(&ptr, &len); + /*const uint32 offsetstart = */ readui32(&ptr, &len); + state->type = type; + + if ((type == 0x92) || (type == 0x93)) + numshaders++; + } // for + } // if + } // for + } // if + } // for + } // if + + // textures... + + if (len < 8) + goto parseEffect_unexpectedEOF; + + const int numtextures = readui32(&ptr, &len); + const int numobjects = readui32(&ptr, &len); // !!! FIXME: "objects" for lack of a better word. + + if (numtextures > 0) + { + siz = sizeof (MOJOSHADER_effectTexture) * numtextures; + retval->textures = m(siz, d); + if (retval->textures == NULL) + goto parseEffect_outOfMemory; + memset(retval->textures, '\0', siz); + + for (i = 0; i < numtextures; i++) + { + if (len < 8) + goto parseEffect_unexpectedEOF; + + MOJOSHADER_effectTexture *texture = &retval->textures[i]; + const uint32 texparam = readui32(&ptr, &len); + const uint32 texsize = readui32(&ptr, &len); + // apparently texsize will pad out to 32 bits. + const uint32 readsize = (((texsize + 3) / 4) * 4); + if (len < readsize) + goto parseEffect_unexpectedEOF; + + texture->param = texparam; + char *str = m(texsize + 1, d); + if (str == NULL) + goto parseEffect_outOfMemory; + memcpy(str, ptr, texsize); + str[texsize] = '\0'; + texture->name = str; + + ptr += readsize; + len -= readsize; + } // for + } // if + + // shaders... + + if (numshaders > 0) + { + siz = sizeof (MOJOSHADER_effectShader) * numshaders; + retval->shaders = (MOJOSHADER_effectShader *) m(siz, d); + if (retval->shaders == NULL) + goto parseEffect_outOfMemory; + memset(retval->shaders, '\0', siz); + + retval->shader_count = numshaders; + + // !!! FIXME: I wonder if we should pull these from offsets and not + // !!! FIXME: count on them all being in a line like this. + for (i = 0; i < numshaders; i++) + { + if (len < 24) + goto parseEffect_unexpectedEOF; + + MOJOSHADER_effectShader *shader = &retval->shaders[i]; + const uint32 technique = readui32(&ptr, &len); + const uint32 pass = readui32(&ptr, &len); + readui32(&ptr, &len); // !!! FIXME: don't know what this does. + readui32(&ptr, &len); // !!! FIXME: don't know what this does (vertex/pixel/geometry?) + readui32(&ptr, &len); // !!! FIXME: don't know what this does. + const uint32 shadersize = readui32(&ptr, &len); + + if (len < shadersize) + goto parseEffect_unexpectedEOF; + + shader->technique = technique; + shader->pass = pass; + shader->shader = MOJOSHADER_parse(profile, ptr, shadersize, + swiz, swizcount, m, f, d); + + // !!! FIXME: check for errors. + + ptr += shadersize; + len -= shadersize; + } // for + } // if + + // !!! FIXME: we parse this, but don't expose the data, yet. + // mappings ... + assert(numshaders <= numobjects); + const uint32 nummappings = numobjects - numshaders; + if (nummappings > 0) + { + for (i = 0; i < nummappings; i++) + { + if (len < 24) + goto parseEffect_unexpectedEOF; + + /*const uint32 magic = */ readui32(&ptr, &len); + /*const uint32 index = */ readui32(&ptr, &len); + readui32(&ptr, &len); // !!! FIXME: what is this field? + readui32(&ptr, &len); // !!! FIXME: what is this field? + /*const uint32 type = */ readui32(&ptr, &len); + const uint32 mapsize = readui32(&ptr, &len); + if (mapsize > 0) + { + const uint32 readsize = (((mapsize + 3) / 4) * 4); + if (len < readsize) + goto parseEffect_unexpectedEOF; + } // if + } // for + } // if + + retval->profile = (char *) m(strlen(profile) + 1, d); + if (retval->profile == NULL) + goto parseEffect_outOfMemory; + strcpy((char *) retval->profile, profile); + + return retval; + + +// !!! FIXME: do something with this. +parseEffect_notAnEffectsFile: +parseEffect_unexpectedEOF: +parseEffect_outOfMemory: + MOJOSHADER_freeEffect(retval); + return &MOJOSHADER_out_of_mem_effect; +} // MOJOSHADER_parseEffect + + +void MOJOSHADER_freeEffect(const MOJOSHADER_effect *_effect) +{ + MOJOSHADER_effect *effect = (MOJOSHADER_effect *) _effect; + if ((effect == NULL) || (effect == &MOJOSHADER_out_of_mem_effect)) + return; // no-op. + + MOJOSHADER_free f = effect->free; + void *d = effect->malloc_data; + int i, j; + + for (i = 0; i < effect->error_count; i++) + { + f((void *) effect->errors[i].error, d); + f((void *) effect->errors[i].filename, d); + } // for + f((void *) effect->errors, d); + + f((void *) effect->profile, d); + + for (i = 0; i < effect->technique_count; i++) + { + MOJOSHADER_effectTechnique *technique = &effect->techniques[i]; + for (j = 0; j < technique->pass_count; j++) + f(technique->passes[j].states, d); + f(technique->passes, d); + } // for + + f(effect->techniques, d); + + for (i = 0; i < effect->texture_count; i++) + f((void *) effect->textures[i].name, d); + f(effect->textures, d); + + for (i = 0; i < effect->shader_count; i++) + MOJOSHADER_freeParseData(effect->shaders[i].shader); + f(effect->shaders, d); + + f(effect, d); +} // MOJOSHADER_freeEffect + +// end of mojoshader_effects.c ... + diff --git a/utils/testparse.c b/utils/testparse.c index 31252a42..0f660b5e 100644 --- a/utils/testparse.c +++ b/utils/testparse.c @@ -39,6 +39,16 @@ static void Free(void *_ptr) #define Free NULL #endif +static inline void do_indent(const unsigned int indent) +{ + unsigned int i; + for (i = 0; i < indent; i++) + printf(" "); +} + +#define INDENT() do { if (indent) { do_indent(indent); } } while (0) + + static const char *shader_type(const MOJOSHADER_shaderType s) { switch (s) @@ -54,33 +64,29 @@ static const char *shader_type(const MOJOSHADER_shaderType s) } // shader_type -static int do_parse(const char *fname, const unsigned char *buf, - const int len, const char *prof) +static void print_shader(const char *fname, const MOJOSHADER_parseData *pd, + unsigned int indent) { - const MOJOSHADER_parseData *pd; - int retval = 0; - - pd = MOJOSHADER_parse(prof, buf, len, NULL, 0, Malloc, Free, NULL); - printf("PROFILE: %s\n", prof); + INDENT(); printf("PROFILE: %s\n", pd->profile); if (pd->error_count > 0) { int i; for (i = 0; i < pd->error_count; i++) { + const MOJOSHADER_error *err = &pd->errors[i]; + INDENT(); printf("%s:%d: ERROR: %s\n", - pd->errors[i].filename ? pd->errors[i].filename : fname, - pd->errors[i].error_position, - pd->errors[i].error); + err->filename ? err->filename : fname, + err->error_position, err->error); } // for } // if else { - retval = 1; - printf("SHADER TYPE: %s\n", shader_type(pd->shader_type)); - printf("VERSION: %d.%d\n", pd->major_ver, pd->minor_ver); - printf("INSTRUCTION COUNT: %d\n", (int) pd->instruction_count); + INDENT(); printf("SHADER TYPE: %s\n", shader_type(pd->shader_type)); + INDENT(); printf("VERSION: %d.%d\n", pd->major_ver, pd->minor_ver); + INDENT(); printf("INSTRUCTION COUNT: %d\n", (int) pd->instruction_count); - printf("ATTRIBUTES:"); + INDENT(); printf("ATTRIBUTES:"); if (pd->attribute_count == 0) printf(" (none.)\n"); else @@ -98,6 +104,7 @@ static int do_parse(const char *fname, const unsigned char *buf, char numstr[16] = { 0 }; if (a->index != 0) snprintf(numstr, sizeof (numstr), "%d", a->index); + INDENT(); printf(" * %s%s", usagenames[(int) a->usage], numstr); if (a->name != NULL) printf(" (\"%s\")", a->name); @@ -105,7 +112,7 @@ static int do_parse(const char *fname, const unsigned char *buf, } // for } // else - printf("CONSTANTS:"); + INDENT(); printf("CONSTANTS:"); if (pd->constant_count == 0) printf(" (none.)\n"); else @@ -116,6 +123,7 @@ static int do_parse(const char *fname, const unsigned char *buf, { static const char *typenames[] = { "float", "int", "bool" }; const MOJOSHADER_constant *c = &pd->constants[i]; + INDENT(); printf(" * %d: %s (", c->index, typenames[(int) c->type]); if (c->type == MOJOSHADER_UNIFORM_FLOAT) { @@ -139,7 +147,7 @@ static int do_parse(const char *fname, const unsigned char *buf, } // for } // else - printf("UNIFORMS:"); + INDENT(); printf("UNIFORMS:"); if (pd->uniform_count == 0) printf(" (none.)\n"); else @@ -160,6 +168,7 @@ static int do_parse(const char *fname, const unsigned char *buf, u->array_count); } // if + INDENT(); printf(" * %d: %s%s%s%s", u->index, constant, arrayof, arrayrange, typenames[(int) u->type]); if (u->name != NULL) @@ -168,7 +177,7 @@ static int do_parse(const char *fname, const unsigned char *buf, } // for } // else - printf("SAMPLERS:"); + INDENT(); printf("SAMPLERS:"); if (pd->sampler_count == 0) printf(" (none.)\n"); else @@ -179,6 +188,7 @@ static int do_parse(const char *fname, const unsigned char *buf, { static const char *typenames[] = { "2d", "cube", "volume" }; const MOJOSHADER_sampler *s = &pd->samplers[i]; + INDENT(); printf(" * %d: %s", s->index, typenames[(int) s->type]); if (s->name != NULL) printf(" (\"%s\")", s->name); @@ -189,14 +199,110 @@ static int do_parse(const char *fname, const unsigned char *buf, if (pd->output != NULL) { int i; + INDENT(); printf("OUTPUT:\n"); + indent++; + INDENT(); for (i = 0; i < pd->output_len; i++) + { putchar((int) pd->output[i]); + if (pd->output[i] == '\n') + INDENT(); + } // for printf("\n"); + indent--; } // if } // else printf("\n\n"); - MOJOSHADER_freeParseData(pd); +} // print_shader + + +static void print_effect(const char *fname, const MOJOSHADER_effect *effect, + const unsigned int indent) +{ + INDENT(); printf("PROFILE: %s\n", effect->profile); + printf("\n"); + if (effect->error_count > 0) + { + int i; + for (i = 0; i < effect->error_count; i++) + { + const MOJOSHADER_error *err = &effect->errors[i]; + INDENT(); + printf("%s:%d: ERROR: %s\n", + err->filename ? err->filename : fname, + err->error_position, err->error); + } // for + } // if + else + { + int i, j, k; + const MOJOSHADER_effectTechnique *technique = effect->techniques; + const MOJOSHADER_effectTexture *texture = effect->textures; + const MOJOSHADER_effectShader *shader = effect->shaders; + + for (i = 0; i < effect->technique_count; i++, technique++) + { + const MOJOSHADER_effectPass *pass = technique->passes; + INDENT(); printf("TECHNIQUE #%d ('%s'):\n", i, technique->name); + for (j = 0; j < technique->pass_count; j++, pass++) + { + const MOJOSHADER_effectState *state = pass->states; + INDENT(); printf(" PASS #%d ('%s'):\n", j, pass->name); + for (k = 0; k < pass->state_count; k++, state++) + { + INDENT(); printf(" STATE 0x%X\n", state->type); + } // for + } // for + printf("\n"); + } // for + + for (i = 0; i < effect->texture_count; i++, texture++) + { + INDENT(); + printf("TEXTURE #%d ('%s'): %u\n", i, + texture->name, texture->param); + } // for + + printf("\n"); + + for (i = 0; i < effect->shader_count; i++, shader++) + { + INDENT(); + printf("SHADER #%d: technique %u, pass %u\n", i, + shader->technique, shader->pass); + print_shader(fname, shader->shader, indent + 1); + } // for + } // else +} // print_effect + + +static int do_parse(const char *fname, const unsigned char *buf, + const int len, const char *prof) +{ + int retval = 0; + + // magic for an effects file (!!! FIXME: I _think_). + if ( (buf[0] == 0x01) && (buf[1] == 0x09) && + (buf[2] == 0xFF) && (buf[3] == 0xFE) ) + { + const MOJOSHADER_effect *effect; + effect = MOJOSHADER_parseEffect(prof, buf, len, 0, 0, Malloc, Free, 0); + retval = (effect->error_count == 0); + printf("EFFECT: %s\n", fname); + print_effect(fname, effect, 1); + MOJOSHADER_freeEffect(effect); + } // if + + else // do it as a regular compiled shader. + { + const MOJOSHADER_parseData *pd; + pd = MOJOSHADER_parse(prof, buf, len, NULL, 0, Malloc, Free, NULL); + retval = (pd->error_count == 0); + printf("SHADER: %s\n", fname); + print_shader(fname, pd, 1); + MOJOSHADER_freeParseData(pd); + } // else return retval; } // do_parse @@ -221,7 +327,6 @@ int main(int argc, char **argv) for (i = 2; i < argc; i++) { FILE *io = fopen(argv[i], "rb"); - printf("FILE: %s\n", argv[i]); if (io == NULL) printf(" ... fopen('%s') failed.\n", argv[i]); else