mojoshader_assembler.c
changeset 946 16fec3a3f687
parent 945 f00ea3986db8
child 947 16af6e2b2ada
--- a/mojoshader_assembler.c	Tue Nov 09 05:05:41 2010 -0500
+++ b/mojoshader_assembler.c	Wed Nov 10 00:52:01 2010 -0500
@@ -54,13 +54,9 @@
     uint32 tokenbuf[16];    // bytecode tokens!
     int tokenbufpos;        // bytecode tokens!
     DestArgInfo dest_arg;
-    uint32 *output;
-    SourcePos *token_to_source;
-    uint8 *ctab;
-    uint32 ctab_len;
-    uint32 ctab_allocation;
-    size_t output_len;
-    size_t output_allocation;
+    Buffer *output;
+    Buffer *token_to_source;
+    Buffer *ctab;
 } Context;
 
 
@@ -228,50 +224,24 @@
     return ctx->tokenval;
 } // nexttoken
 
-// !!! FIXME: hashmap?
-static inline void add_token_sourcepos(Context *ctx, const size_t idx)
-{
-    unsigned int pos = 0;
-    const char *fname = preprocessor_sourcepos(ctx->preprocessor, &pos);
-    ctx->token_to_source[idx].line = pos;
-    ctx->token_to_source[idx].filename = fname;  // cached in preprocessor!
-} // add_token_sourcepos
 
-
-// !!! FIXME: use BufferList.
 static void output_token_noswap(Context *ctx, const uint32 token)
 {
-    if (isfail(ctx))
-        return;
-
-    if (ctx->output_len >= ctx->output_allocation)
+    if (!isfail(ctx))
     {
-        const size_t output_alloc_bump = 1024;  // that's tokens, not bytes.
-        const size_t newsize = ctx->output_allocation + output_alloc_bump;
-        void *ptr;
+        buffer_append(ctx->output, &token, sizeof (token));
 
-        ptr = Malloc(ctx, newsize * sizeof (uint32));
-        if (ptr == NULL)
-            return;
-        if (ctx->output_len > 0)
-            memcpy(ptr, ctx->output, ctx->output_len * sizeof (uint32));
-        Free(ctx, ctx->output);
-        ctx->output = (uint32 *) ptr;
-
-        ptr = Malloc(ctx, newsize * sizeof (SourcePos));
-        if (ptr == NULL)
-            return;
-        if (ctx->output_len > 0)
-            memcpy(ptr, ctx->token_to_source, ctx->output_len * sizeof (SourcePos));
-        Free(ctx, ctx->token_to_source);
-        ctx->token_to_source = (SourcePos *) ptr;
-
-        ctx->output_allocation = newsize;
+        // We only need a list of these that grows throughout processing, and
+        //  is flattened for reference at the end of the run, so we use a
+        //  Buffer. It's sneaky!
+        unsigned int pos = 0;
+        const char *fname = preprocessor_sourcepos(ctx->preprocessor, &pos);
+        SourcePos srcpos;
+        memset(&srcpos, '\0', sizeof (SourcePos));
+        srcpos.line = pos;
+        srcpos.filename = fname;  // cached in preprocessor!
+        buffer_append(ctx->token_to_source, &srcpos, sizeof (SourcePos));
     } // if
-
-    ctx->output[ctx->output_len] = token;
-    add_token_sourcepos(ctx, ctx->output_len);
-    ctx->output_len++;
 } // output_token_noswap
 
 
@@ -289,8 +259,6 @@
     {
         const uint32 tokencount = (len / 4) + ((len % 4) ? 1 : 0);
         output_token(ctx, 0xFFFE | (tokencount << 16));
-        // !!! FIXME: we can probably just use use modulo and do this without
-        // !!! FIXME:  a tight loop with BufferList.
         while (len >= 4)
         {
             output_token_noswap(ctx, *((const uint32 *) buf));
@@ -1431,6 +1399,22 @@
 } // parse_token
 
 
+static void destroy_context(Context *ctx)
+{
+    if (ctx != NULL)
+    {
+        MOJOSHADER_free f = ((ctx->free != NULL) ? ctx->free : MOJOSHADER_internal_free);
+        void *d = ctx->malloc_data;
+        preprocessor_end(ctx->preprocessor);
+        errorlist_destroy(ctx->errors);
+        buffer_destroy(ctx->ctab);
+        buffer_destroy(ctx->token_to_source);
+        buffer_destroy(ctx->output);
+        f(ctx, d);
+    } // if
+} // destroy_context
+
+
 static Context *build_context(const char *filename,
                               const char *source, unsigned int sourcelen,
                               const MOJOSHADER_preprocessorDefine *defines,
@@ -1454,12 +1438,19 @@
     ctx->malloc_data = d;
     ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED;
 
+    const size_t outblk = sizeof (uint32) * 4 * 64; // 64 4-token instrs.
+    ctx->output = buffer_create(outblk, MallocBridge, FreeBridge, ctx);
+    if (ctx->output == NULL)
+        goto build_context_failed;
+
+    const size_t mapblk = sizeof (SourcePos) * 4 * 64; // 64 * 4-tokens.
+    ctx->token_to_source = buffer_create(mapblk, MallocBridge, FreeBridge, ctx);
+    if (ctx->token_to_source == NULL)
+        goto build_context_failed;
+
     ctx->errors = errorlist_create(MallocBridge, FreeBridge, ctx);
     if (ctx->errors == NULL)
-    {
-        f(ctx, d);
-        return NULL;
-    } // if
+        goto build_context_failed;
 
     ctx->preprocessor = preprocessor_start(filename, source, sourcelen,
                                            include_open, include_close,
@@ -1467,37 +1458,16 @@
                                            MallocBridge, FreeBridge, ctx);
 
     if (ctx->preprocessor == NULL)
-    {
-        errorlist_destroy(ctx->errors);
-        f(ctx, d);
-        return NULL;
-    } // if
+        goto build_context_failed;
 
     return ctx;
+
+build_context_failed:  // ctx is allocated and zeroed before this is called.
+    destroy_context(ctx);
+    return NULL;
 } // build_context
 
 
-static void destroy_context(Context *ctx)
-{
-    if (ctx != NULL)
-    {
-        MOJOSHADER_free f = ((ctx->free != NULL) ? ctx->free : MOJOSHADER_internal_free);
-        void *d = ctx->malloc_data;
-        if (ctx->errors != NULL)
-            errorlist_destroy(ctx->errors);
-        if (ctx->preprocessor != NULL)
-            preprocessor_end(ctx->preprocessor);
-        if (ctx->output != NULL)
-            f(ctx->output, d);
-        if (ctx->token_to_source != NULL)
-            f(ctx->token_to_source, d);
-        if (ctx->ctab != NULL)
-            f(ctx->ctab, d);
-        f(ctx, d);
-    } // if
-} // destroy_context
-
-
 static const MOJOSHADER_parseData *build_failed_assembly(Context *ctx)
 {
     assert(isfail(ctx));
@@ -1517,8 +1487,10 @@
 
     retval->error_count = errorlist_count(ctx->errors);
     retval->errors = errorlist_flatten(ctx->errors);
+
     if (ctx->out_of_memory)
     {
+        Free(ctx, retval->errors);
         Free(ctx, retval);
         return &MOJOSHADER_out_of_mem_data;
     } // if
@@ -1529,48 +1501,17 @@
 
 static uint32 add_ctab_bytes(Context *ctx, const uint8 *bytes, const size_t len)
 {
+    if (isfail(ctx))
+        return 0;
+
     const size_t extra = CTAB_SIZE + sizeof (uint32);
-    if (len <= (ctx->ctab_len - extra))
-    {
-        uint8 *ptr = ctx->ctab + extra;
-        if (len == 0)
-            return ( (uint32) (ptr - ctx->ctab) ) - sizeof (uint32);
-        else if ((len == 1) && ((ptr = (uint8 *) memchr(ptr, bytes[0], ctx->ctab_len - len)) != NULL))
-            return ( (uint32) (ptr - ctx->ctab) ) - sizeof (uint32);
-        else  // search for the string of bytes...
-        {
-            while ((ptr = (uint8 *) memchr(ptr, bytes[0], ctx->ctab_len - len)) != NULL)
-            {
-                if (memcmp(ptr, bytes, len) == 0)  // this is it?
-                    return ( (uint32) (ptr - ctx->ctab) ) - sizeof (uint32);
-                ptr++;  // !!! FIXME: should this be "ptr += len;"  ?
-            } // while
-        } // else
-    } // if
+    const ssize_t pos = buffer_find(ctx->ctab, extra, bytes, len);
+    if (pos >= 0)  // blob is already in here.
+        return ((uint32) pos) - sizeof (uint32);
 
     // add it to the byte pile...
-
-    // verify allocation.
-    const size_t newsize = (ctx->ctab_len + len);
-    if (ctx->ctab_allocation < newsize)
-    {
-        const size_t additional = 4 * 1024;
-        while (ctx->ctab_allocation < newsize)
-            ctx->ctab_allocation += additional;
-        void *ptr = Malloc(ctx, ctx->ctab_allocation);
-        if (ptr == NULL)
-            return 0;
-        if (ctx->ctab != NULL)
-        {
-            memcpy(ptr, ctx->ctab, ctx->ctab_len);
-            Free(ctx, ctx->ctab);
-        } // if
-        ctx->ctab = (uint8 *) ptr;
-    } // if
-
-    const uint32 retval = ctx->ctab_len - sizeof (uint32);
-    memcpy(ctx->ctab + ctx->ctab_len, bytes, len);
-    ctx->ctab_len += len;
+    const uint32 retval = ((uint32) buffer_size(ctx->ctab)) - sizeof (uint32);
+    buffer_append(ctx->ctab, bytes, len);
     return retval;
 } // add_ctab_bytes
 
@@ -1656,10 +1597,20 @@
 static void output_ctab(Context *ctx, const MOJOSHADER_symbol *symbols,
                         unsigned int symbol_count, const char *creator)
 {
-    ctx->ctab_len = CTAB_SIZE + sizeof (uint32);
+    const size_t tablelen = CTAB_SIZE + sizeof (uint32);
+
+    ctx->ctab = buffer_create(256, MallocBridge, FreeBridge, ctx);
+    if (ctx->ctab == NULL)
+        return;  // out of memory.
 
-    uint8 bytes[CTAB_SIZE + sizeof (uint32)];
-    uint32 *table = (uint32 *) bytes;
+    uint32 *table = (uint32 *) buffer_reserve(ctx->ctab, tablelen);
+    if (table == NULL)
+    {
+        buffer_destroy(ctx->ctab);
+        ctx->ctab = NULL;
+        return;  // out of memory.
+    } // if
+
     *(table++) = SWAP32(CTAB_ID);
     *(table++) = SWAP32(CTAB_SIZE);
     *(table++) = SWAP32(add_ctab_string(ctx, creator));
@@ -1668,13 +1619,17 @@
     *(table++) = SWAP32(add_ctab_info(ctx, symbols, symbol_count));
     *(table++) = SWAP32(0);  // build flags.
     *(table++) = SWAP32(add_ctab_string(ctx, ""));  // !!! FIXME: target?
-    memcpy(ctx->ctab, bytes, sizeof (bytes));
-    output_comment_bytes(ctx, ctx->ctab, ctx->ctab_len);
 
-    Free(ctx, ctx->ctab);
+    const size_t ctablen = buffer_size(ctx->ctab);
+    uint8 *buf = (uint8 *) buffer_flatten(ctx->ctab);
+    if (buf != NULL)
+    {
+        output_comment_bytes(ctx, buf, ctablen);
+        Free(ctx, buf);
+    } // if
+
+    buffer_destroy(ctx->ctab);
     ctx->ctab = NULL;
-    ctx->ctab_len = 0;
-    ctx->ctab_allocation = 0;
 } // output_ctab
 
 
@@ -1703,6 +1658,80 @@
 } // output_comments
 
 
+static const MOJOSHADER_parseData *build_final_assembly(Context *ctx)
+{
+    if (isfail(ctx))
+        return build_failed_assembly(ctx);
+
+    // get the final bytecode!
+    const unsigned int output_len = (unsigned int) buffer_size(ctx->output);
+    unsigned char *bytecode = buffer_flatten(ctx->output);
+    buffer_destroy(ctx->output);
+    ctx->output = NULL;
+
+    if (bytecode == NULL)
+        return build_failed_assembly(ctx);
+
+    // This validates the shader; there are lots of things that are
+    //  invalid, but will successfully parse in the assembler,
+    //  generating bad bytecode; this will catch them without us
+    //  having to duplicate most of the validation here.
+    // It also saves us the trouble of duplicating all the other work,
+    //  like setting up the uniforms list, etc.
+    MOJOSHADER_parseData *retval = (MOJOSHADER_parseData *)
+                            MOJOSHADER_parse(MOJOSHADER_PROFILE_BYTECODE,
+                                    bytecode, output_len, NULL, 0,
+                                    ctx->malloc, ctx->free, ctx->malloc_data);
+    Free(ctx, bytecode);
+
+    SourcePos *token_to_src = NULL;
+    if (retval->error_count > 0)
+        token_to_src = (SourcePos *) buffer_flatten(ctx->token_to_source);
+    buffer_destroy(ctx->token_to_source);
+    ctx->token_to_source = NULL;
+
+    if (retval->error_count > 0)
+    {
+        if (token_to_src == NULL)
+        {
+            assert(ctx->out_of_memory);
+            MOJOSHADER_freeParseData(retval);
+            return build_failed_assembly(ctx);
+        } // if
+
+        // on error, map the bytecode back to a line number.
+        int i;
+        for (i = 0; i < retval->error_count; i++)
+        {
+            MOJOSHADER_error *error = &retval->errors[i];
+            if (error->error_position >= 0)
+            {
+                assert(retval != &MOJOSHADER_out_of_mem_data);
+                assert((error->error_position % sizeof (uint32)) == 0);
+
+                const size_t pos = error->error_position / sizeof(uint32);
+                if (pos >= output_len)
+                    error->error_position = -1;  // oh well.
+                else
+                {
+                    const SourcePos *srcpos = &token_to_src[pos];
+                    Free(ctx, (void *) error->filename);
+                    char *fname = NULL;
+                    if (srcpos->filename != NULL)
+                        fname = StrDup(ctx, srcpos->filename);
+                    error->error_position = srcpos->line;
+                    error->filename = fname;  // may be NULL, that's okay.
+                } // else
+            } // if
+        } // for
+
+        Free(ctx, token_to_src);
+    } // if
+
+    return retval;
+} // build_final_assembly
+
+
 // API entry point...
 
 const MOJOSHADER_parseData *MOJOSHADER_assemble(const char *filename,
@@ -1716,7 +1745,7 @@
                              MOJOSHADER_includeClose include_close,
                              MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
 {
-    MOJOSHADER_parseData *retval = NULL;
+    const MOJOSHADER_parseData *retval = NULL;
     Context *ctx = NULL;
 
     if ( ((m == NULL) && (f != NULL)) || ((m != NULL) && (f == NULL)) )
@@ -1739,49 +1768,7 @@
 
     output_token(ctx, 0x0000FFFF);   // end token always 0x0000FFFF.
 
-    if (isfail(ctx))
-        retval = (MOJOSHADER_parseData *) build_failed_assembly(ctx);
-    else
-    {
-        // This validates the shader; there are lots of things that are
-        //  invalid, but will successfully parse in the assembler, generating
-        //  bad bytecode; this will catch them without us having to
-        //  duplicate most of the validation here.
-        // It also saves us the trouble of duplicating all the other work,
-        //  like setting up the uniforms list, etc.
-        retval = (MOJOSHADER_parseData *)
-                        MOJOSHADER_parse(MOJOSHADER_PROFILE_BYTECODE,
-                                      (const unsigned char *) ctx->output,
-                                      ctx->output_len * sizeof (uint32),
-                                      NULL, 0, m, f, d);
-
-        // on error, map the bytecode back to a line number.
-        int i;
-        for (i = 0; i < retval->error_count; i++)
-        {
-            MOJOSHADER_error *error = &retval->errors[i];
-            if (error->error_position >= 0)
-            {
-                assert(retval != &MOJOSHADER_out_of_mem_data);
-                assert((error->error_position % sizeof (uint32)) == 0);
-
-                const size_t pos = error->error_position / sizeof (uint32);
-                if (pos >= ctx->output_len)
-                    error->error_position = -1;  // oh well.
-                else
-                {
-                    const SourcePos *srcpos = &ctx->token_to_source[pos];
-                    Free(ctx, (void *) error->filename);
-                    char *fname = NULL;
-                    if (srcpos->filename != NULL)
-                        fname = StrDup(ctx, srcpos->filename);
-                    error->error_position = srcpos->line;
-                    error->filename = fname;  // may be NULL, that's okay.
-                } // else
-            } // if
-        } // for
-    } // if
-
+    retval = build_final_assembly(ctx);
     destroy_context(ctx);
     return retval;
 } // MOJOSHADER_assemble