Paying off more technical debt: unified growable buffers into one place.
authorRyan C. Gordon <icculus@icculus.org>
Tue, 09 Nov 2010 05:00:03 -0500
changeset 944 9f9fa9650772
parent 943 775cd2ac324b
child 945 f00ea3986db8
Paying off more technical debt: unified growable buffers into one place.
mojoshader.c
mojoshader_common.c
mojoshader_internal.h
mojoshader_preprocessor.c
--- a/mojoshader.c	Mon Nov 08 19:06:56 2010 -0500
+++ b/mojoshader.c	Tue Nov 09 05:00:03 2010 -0500
@@ -13,24 +13,6 @@
 #define __MOJOSHADER_INTERNAL__ 1
 #include "mojoshader_internal.h"
 
-// !!! FIXME: replace this with BufferList from the preprocessor.
-// !!! FIXME: we don't need these as a list of strings, we just need a
-// !!! FIXME:  growable buffer we can flatten at the end.
-// A simple linked list of strings, so we can build the final output without
-//  realloc()ing for each new line, and easily insert lines into the middle
-//  of the output without much trouble.
-typedef struct OutputListNode
-{
-    char *str;
-    struct OutputListNode *next;
-} OutputListNode;
-
-typedef struct OutputList
-{
-    OutputListNode head;
-    OutputListNode *tail;
-} OutputList;
-
 typedef struct ConstantsList
 {
     MOJOSHADER_constant constant;
@@ -94,19 +76,17 @@
     MOJOSHADER_parsePhase parse_phase;
     const MOJOSHADER_swizzle *swizzles;
     unsigned int swizzles_count;
-    OutputList *output;
-    OutputList preflight;
-    OutputList globals;
-    OutputList helpers;
-    OutputList subroutines;
-    OutputList mainline_intro;
-    OutputList mainline;
-    OutputList ignore;
-    OutputList *output_stack[2];
-    uint8 *output_bytes;  // can be used instead of the OutputLists.
+    Buffer *output;
+    Buffer *preflight;
+    Buffer *globals;
+    Buffer *helpers;
+    Buffer *subroutines;
+    Buffer *mainline_intro;
+    Buffer *mainline;
+    Buffer *ignore;
+    Buffer *output_stack[2];
     int indent_stack[2];
     int output_stack_len;
-    int output_len; // total strlen; prevents walking the lists just to malloc.
     int indent;
     const char *shader_type_str;
     const char *endline;
@@ -312,13 +292,28 @@
 
 // jump between output sections in the context...
 
-static inline void push_output(Context *ctx, OutputList *section)
+static int set_output(Context *ctx, Buffer **section)
+{
+    // only create output sections on first use.
+    if (*section == NULL)
+    {
+        *section = buffer_create(256, MallocBridge, FreeBridge, ctx);
+        if (*section == NULL)
+            return 0;
+    } // if
+
+    ctx->output = *section;
+    return 1;
+} // set_output
+
+static void push_output(Context *ctx, Buffer **section)
 {
     assert(ctx->output_stack_len < (int) (STATICARRAYLEN(ctx->output_stack)));
     ctx->output_stack[ctx->output_stack_len] = ctx->output;
     ctx->indent_stack[ctx->output_stack_len] = ctx->indent;
     ctx->output_stack_len++;
-    ctx->output = section;
+    if (!set_output(ctx, section))
+        return;
     ctx->indent = 0;
 } // push_output
 
@@ -416,59 +411,32 @@
 static void output_line(Context *ctx, const char *fmt, ...) ISPRINTF(2,3);
 static void output_line(Context *ctx, const char *fmt, ...)
 {
-    OutputListNode *item = NULL;
-
-    if (isfail(ctx) || ctx->out_of_memory)
+    assert(ctx->output != NULL);
+    if (isfail(ctx))
         return;  // we failed previously, don't go on...
 
-    char scratch[128];
-
     const int indent = ctx->indent;
     if (indent > 0)
-        memset(scratch, '\t', indent);
+    {
+        char *indentbuf = (char *) alloca(indent);
+        memset(indentbuf, '\t', indent);
+        buffer_append(ctx->output, indentbuf, indent);
+    } // if
 
     va_list ap;
     va_start(ap, fmt);
-    const int len = vsnprintf(scratch+indent, sizeof (scratch)-indent, fmt, ap) + indent;
+    buffer_append_va(ctx->output, fmt, ap);
     va_end(ap);
 
-    item = (OutputListNode *) Malloc(ctx, sizeof (OutputListNode));
-    if (item == NULL)
-        return;
-
-    item->str = (char *) Malloc(ctx, len + 1);
-    if (item->str == NULL)
-    {
-        Free(ctx, item);
-        return;
-    } // if
-
-    // If we overflowed our scratch buffer, that's okay. We were going to
-    //  allocate anyhow...the scratch buffer just lets us avoid a second
-    //  run of vsnprintf().
-    if (len < sizeof (scratch))
-        strcpy(item->str, scratch);  // copy it over.
-    else
-    {
-        if (indent > 0)
-            memset(item->str, '\t', indent);
-        va_start(ap, fmt);
-        vsnprintf(item->str+indent, len + 1, fmt, ap);  // rebuild it.
-        va_end(ap);
-    } // else
-
-    item->next = NULL;
-
-    ctx->output->tail->next = item;
-    ctx->output->tail = item;
-    ctx->output_len += len + ctx->endline_len;
+    buffer_append(ctx->output, ctx->endline, ctx->endline_len);
 } // output_line
 
 
-// this is just to stop gcc whining.
 static inline void output_blank_line(Context *ctx)
 {
-    output_line(ctx, "%s", "");
+    assert(ctx->output != NULL);
+    if (!isfail(ctx))
+        buffer_append(ctx->output, ctx->endline, ctx->endline_len);
 } // output_blank_line
 
 
@@ -1517,17 +1485,14 @@
 static void emit_BYTECODE_start(Context *ctx, const char *profilestr)
 {
     // just copy the whole token stream and make all other emitters no-ops.
-    ctx->output_len = (ctx->tokencount * sizeof (uint32));
-    ctx->output_bytes = (uint8 *) Malloc(ctx, ctx->output_len);
-    if (ctx->output_bytes != NULL)
-        memcpy(ctx->output_bytes, ctx->tokens, ctx->output_len);
+    if (set_output(ctx, &ctx->mainline))
+    {
+        const size_t len = ctx->tokencount * sizeof (uint32);
+        buffer_append(ctx->mainline, (const char *) ctx->tokens, len);
+    } // if
 } // emit_BYTECODE_start
 
-static void emit_BYTECODE_end(Context *ctx)
-{
-    // no-op in this profile.
-} // emit_BYTECODE_end
-
+static void emit_BYTECODE_end(Context *ctx) {}
 static void emit_BYTECODE_phase(Context *ctx) {}
 static void emit_BYTECODE_finalize(Context *ctx) {}
 static void emit_BYTECODE_global(Context *ctx, RegisterType t, int n) {}
@@ -2109,8 +2074,6 @@
         return;
     } // if
 
-    ctx->output = &ctx->preflight;
-
     if (strcmp(profilestr, MOJOSHADER_PROFILE_GLSL) == 0)
         /* no-op. */ ;
 
@@ -2118,7 +2081,9 @@
     else if (strcmp(profilestr, MOJOSHADER_PROFILE_GLSL120) == 0)
     {
         ctx->profile_supports_glsl120 = 1;
+        push_output(ctx, &ctx->preflight);
         output_line(ctx, "#version 120");
+        pop_output(ctx);
     } // else if
     #endif
 
@@ -2128,10 +2093,12 @@
         return;
     } // else
 
-    ctx->output = &ctx->mainline_intro;
+    push_output(ctx, &ctx->mainline_intro);
     output_line(ctx, "void main()");
     output_line(ctx, "{");
-    ctx->output = &ctx->mainline;
+    pop_output(ctx);
+
+    set_output(ctx, &ctx->mainline);
     ctx->indent++;
 } // emit_GLSL_start
 
@@ -2915,7 +2882,7 @@
     ctx->indent--;
     output_line(ctx, "}");
     output_blank_line(ctx);
-    ctx->output = &ctx->subroutines;
+    set_output(ctx, &ctx->subroutines);
 } // emit_GLSL_RET
 
 static void emit_GLSL_ENDLOOP(Context *ctx)
@@ -2931,13 +2898,13 @@
     char src0[64]; make_GLSL_srcarg_string_masked(ctx, 0, src0, sizeof (src0));
     const int label = ctx->source_args[0].regnum;
     RegisterList *reg = reglist_find(&ctx->used_registers, REG_TYPE_LABEL, label);
-    assert(ctx->output == &ctx->subroutines);  // not mainline, etc.
+    assert(ctx->output == ctx->subroutines);  // not mainline, etc.
     assert(ctx->indent == 0);  // we shouldn't be in the middle of a function.
 
     // MSDN specs say CALL* has to come before the LABEL, so we know if we
     //  can ditch the entire function here as unused.
     if (reg == NULL)
-        ctx->output = &ctx->ignore;  // Func not used. Parse, but don't output.
+        set_output(ctx, &ctx->ignore);  // Func not used. Parse, but don't output.
 
     // !!! FIXME: it would be nice if we could determine if a function is
     // !!! FIXME:  only called once and, if so, forcibly inline it.
@@ -3911,7 +3878,7 @@
         return;
     } // if
 
-    ctx->output = &ctx->globals;
+    set_output(ctx, &ctx->globals);
 
     if (strcmp(profilestr, MOJOSHADER_PROFILE_ARB1) == 0)
         output_line(ctx, "!!ARB%s1.0", shader_str);
@@ -3948,7 +3915,7 @@
         failf(ctx, "Profile '%s' unsupported or unknown.", profilestr);
     } // else
 
-    ctx->output = &ctx->mainline;
+    set_output(ctx, &ctx->mainline);
 } // emit_ARB1_start
 
 static void emit_ARB1_end(Context *ctx)
@@ -4531,7 +4498,7 @@
     //  just end up throwing all this code out.
     if (support_nv2(ctx))  // no branching in stock ARB1.
         output_line(ctx, "RET;");
-    ctx->output = &ctx->mainline;  // in case we were ignoring this function.
+    set_output(ctx, &ctx->mainline); // in case we were ignoring this function.
 } // emit_ARB1_RET
 
 
@@ -4548,7 +4515,7 @@
     // MSDN specs say CALL* has to come before the LABEL, so we know if we
     //  can ditch the entire function here as unused.
     if (reg == NULL)
-        ctx->output = &ctx->ignore;  // Func not used. Parse, but don't output.
+        set_output(ctx, &ctx->ignore);  // Func not used. Parse, but don't output.
 
     // !!! FIXME: it would be nice if we could determine if a function is
     // !!! FIXME:  only called once and, if so, forcibly inline it.
@@ -7017,14 +6984,6 @@
     ctx->swizzles_count = swizcount;
     ctx->endline = ENDLINE_STR;
     ctx->endline_len = strlen(ctx->endline);
-    ctx->preflight.tail = &ctx->preflight.head;
-    ctx->globals.tail = &ctx->globals.head;
-    ctx->helpers.tail = &ctx->helpers.head;
-    ctx->subroutines.tail = &ctx->subroutines.head;
-    ctx->mainline_intro.tail = &ctx->mainline_intro.head;
-    ctx->mainline.tail = &ctx->mainline.head;
-    ctx->ignore.tail = &ctx->ignore.head;
-    ctx->output = &ctx->mainline;
     ctx->last_address_reg_component = -1;
     ctx->parse_phase = MOJOSHADER_PARSEPHASE_NOTSTARTED;
 
@@ -7035,6 +6994,13 @@
         return NULL;
     } // if
 
+    if (!set_output(ctx, &ctx->mainline))
+    {
+        errorlist_destroy(ctx->errors);
+        f(ctx, d);
+        return NULL;
+    } // if
+
     const int profileid = find_profile_id(profile);
     ctx->profileid = profileid;
     if (profileid >= 0)
@@ -7046,19 +7012,6 @@
 } // build_context
 
 
-static void free_output_list(MOJOSHADER_free f, void *d, OutputListNode *item)
-{
-    while (item != NULL)
-    {
-        OutputListNode *next = item->next;
-        if (item->str != NULL)
-            f(item->str, d);
-        f(item, d);
-        item = next;
-    } // while
-} // free_output_list
-
-
 static void free_constants_list(MOJOSHADER_free f, void *d, ConstantsList *item)
 {
     while (item != NULL)
@@ -7087,15 +7040,13 @@
     {
         MOJOSHADER_free f = ((ctx->free != NULL) ? ctx->free : MOJOSHADER_internal_free);
         void *d = ctx->malloc_data;
-        if (ctx->output_bytes != NULL)
-            f(d, ctx->output_bytes);
-        free_output_list(f, d, ctx->preflight.head.next);
-        free_output_list(f, d, ctx->globals.head.next);
-        free_output_list(f, d, ctx->helpers.head.next);
-        free_output_list(f, d, ctx->subroutines.head.next);
-        free_output_list(f, d, ctx->mainline_intro.head.next);
-        free_output_list(f, d, ctx->mainline.head.next);
-        free_output_list(f, d, ctx->ignore.head.next);
+        buffer_destroy(ctx->preflight);
+        buffer_destroy(ctx->globals);
+        buffer_destroy(ctx->helpers);
+        buffer_destroy(ctx->subroutines);
+        buffer_destroy(ctx->mainline_intro);
+        buffer_destroy(ctx->mainline);
+        buffer_destroy(ctx->ignore);
         free_constants_list(f, d, ctx->constants);
         free_reglist(f, d, ctx->used_registers.next);
         free_reglist(f, d, ctx->defined_registers.next);
@@ -7109,48 +7060,15 @@
 } // destroy_context
 
 
-static void append_list(char **_wptr, const char *endline,
-                        const size_t endline_len, OutputListNode *item)
-{
-    char *wptr = *_wptr;
-    while (item != NULL)
-    {
-        const size_t len = strlen(item->str);
-        memcpy(wptr, item->str, len);
-        wptr += len;
-        memcpy(wptr, endline, endline_len);
-        wptr += endline_len;
-        item = item->next;
-    } // while
-    *wptr = '\0';
-    *_wptr = wptr;
-} // append_list
-
-
-static char *build_output(Context *ctx)
-{
-    // add a byte for the null terminator if we're doing text output.
-    const int plusbytes = (ctx->output_bytes == NULL) ? 1 : 0;
-    char *retval = (char *) Malloc(ctx, ctx->output_len + plusbytes);
-    if (retval != NULL)
-    {
-        const char *endl = ctx->endline;
-        const size_t endllen = ctx->endline_len;
-        char *wptr = retval;
-        if (ctx->output_bytes != NULL)
-            memcpy(retval, ctx->output_bytes, ctx->output_len);
-        else
-        {
-            append_list(&wptr, endl, endllen, ctx->preflight.head.next);
-            append_list(&wptr, endl, endllen, ctx->globals.head.next);
-            append_list(&wptr, endl, endllen, ctx->helpers.head.next);
-            append_list(&wptr, endl, endllen, ctx->subroutines.head.next);
-            append_list(&wptr, endl, endllen, ctx->mainline_intro.head.next);
-            append_list(&wptr, endl, endllen, ctx->mainline.head.next);
-            // don't append ctx->ignore ... that's why it's called "ignore"
-        } // else
-    } // if
-
+static char *build_output(Context *ctx, size_t *len)
+{
+    // add a byte for a null terminator.
+    Buffer *buffers[] = {
+        ctx->preflight, ctx->globals, ctx->helpers,
+        ctx->subroutines, ctx->mainline_intro, ctx->mainline
+        // don't append ctx->ignore ... that's why it's called "ignore"
+    };
+    char *retval = buffer_merge(buffers, STATICARRAYLEN(buffers), len);
     return retval;
 } // build_output
 
@@ -7414,6 +7332,7 @@
     MOJOSHADER_swizzle *swizzles = NULL;
     MOJOSHADER_error *errors = NULL;
     MOJOSHADER_parseData *retval = NULL;
+    size_t output_len = 0;
     int attribute_count = 0;
 
     if (ctx->out_of_memory)
@@ -7426,7 +7345,7 @@
     memset(retval, '\0', sizeof (MOJOSHADER_parseData));
 
     if (!isfail(ctx))
-        output = build_output(ctx);
+        output = build_output(ctx, &output_len);
 
     if (!isfail(ctx))
         constants = build_constants(ctx);
@@ -7500,7 +7419,7 @@
     {
         retval->profile = ctx->profile->name;
         retval->output = output;
-        retval->output_len = ctx->output_len;
+        retval->output_len = (int) output_len;
         retval->instruction_count = ctx->instruction_count;
         retval->shader_type = ctx->shader_type;
         retval->major_ver = (int) ctx->major_ver;
--- a/mojoshader_common.c	Mon Nov 08 19:06:56 2010 -0500
+++ b/mojoshader_common.c	Tue Nov 09 05:00:03 2010 -0500
@@ -533,5 +533,227 @@
 } // errorlist_destroy
 
 
+typedef struct BufferBlock
+{
+    char *data;
+    size_t bytes;
+    struct BufferBlock *next;
+} BufferBlock;
+
+struct Buffer
+{
+    size_t total_bytes;
+    BufferBlock *head;
+    BufferBlock *tail;
+    size_t block_size;
+    MOJOSHADER_malloc m;
+    MOJOSHADER_free f;
+    void *d;
+};
+
+Buffer *buffer_create(size_t blksz, MOJOSHADER_malloc m,
+                      MOJOSHADER_free f, void *d)
+{
+    Buffer *buffer = (Buffer *) m(sizeof (Buffer), d);
+    if (buffer != NULL)
+    {
+        memset(buffer, '\0', sizeof (Buffer));
+        buffer->block_size = blksz;
+        buffer->m = m;
+        buffer->f = f;
+        buffer->d = d;
+    } // if
+    return buffer;
+} // buffer_create
+
+int buffer_append(Buffer *buffer, const char *data, size_t len)
+{
+    // note that we make the blocks bigger than blocksize when we have enough
+    //  data to overfill a fresh block, to reduce allocations.
+    const size_t blocksize = buffer->block_size;
+
+    if (len == 0)
+        return 1;
+
+    if (buffer->tail != NULL)
+    {
+        const size_t tailbytes = buffer->tail->bytes;
+        const size_t avail = (tailbytes >= blocksize) ? 0 : blocksize - tailbytes;
+        const size_t cpy = (avail > len) ? len : avail;
+        if (cpy > 0)
+        {
+            memcpy(buffer->tail->data + tailbytes, data, cpy);
+            len -= cpy;
+            data += cpy;
+            buffer->tail->bytes += cpy;
+            buffer->total_bytes += cpy;
+            assert(buffer->tail->bytes <= blocksize);
+        } // if
+    } // if
+
+    if (len > 0)
+    {
+        assert((!buffer->tail) || (buffer->tail->bytes == blocksize));
+        const size_t bytecount = len > blocksize ? len : blocksize;
+        const size_t malloc_len = sizeof (BufferBlock) + bytecount;
+        BufferBlock *item = (BufferBlock *) buffer->m(malloc_len, buffer->d);
+        if (item == NULL)
+            return 0;
+
+        item->data = ((char *) item) + sizeof (BufferBlock);
+        item->bytes = len;
+        item->next = NULL;
+        if (buffer->tail != NULL)
+            buffer->tail->next = item;
+        else
+            buffer->head = item;
+        buffer->tail = item;
+
+        memcpy(item->data, data, len);
+        buffer->total_bytes += len;
+    } // if
+
+    return 1;
+} // buffer_append
+
+int buffer_append_fmt(Buffer *buffer, const char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    const int retval = buffer_append_va(buffer, fmt, ap);
+    va_end(ap);
+    return retval;
+} // buffer_append_fmt
+
+int buffer_append_va(Buffer *buffer, const char *fmt, va_list va)
+{
+    char scratch[128];
+
+    va_list ap;
+    va_copy(ap, va);
+    const int len = vsnprintf(scratch, sizeof (scratch), fmt, ap);
+    va_end(ap);
+
+    // If we overflowed our scratch buffer, heap allocate and try again.
+
+    if (len == 0)
+        return 1;  // nothing to do.
+    else if (len < sizeof (scratch))
+        return buffer_append(buffer, scratch, len);
+
+    char *buf = (char *) buffer->m(len + 1, buffer->d);
+    if (buf == NULL)
+        return 0;
+    va_copy(ap, va);
+    vsnprintf(buf, len + 1, fmt, ap);  // rebuild it.
+    va_end(ap);
+    const int retval = buffer_append(buffer, scratch, len);
+    buffer->f(buf, buffer->d);
+    return retval;
+} // buffer_append_va
+
+size_t buffer_size(Buffer *buffer)
+{
+    return buffer->total_bytes;
+} // buffer_size
+
+void buffer_empty(Buffer *buffer)
+{
+    BufferBlock *item = buffer->head;
+    while (item != NULL)
+    {
+        BufferBlock *next = item->next;
+        buffer->f(item, buffer->d);
+        item = next;
+    } // while
+    buffer->head = buffer->tail = NULL;
+    buffer->total_bytes = 0;
+} // buffer_empty
+
+char *buffer_flatten(Buffer *buffer)
+{
+    char *retval = (char *) buffer->m(buffer->total_bytes + 1, buffer->d);
+    if (retval == NULL)
+        return NULL;
+    BufferBlock *item = buffer->head;
+    char *ptr = retval;
+    while (item != NULL)
+    {
+        BufferBlock *next = item->next;
+        memcpy(ptr, item->data, item->bytes);
+        ptr += item->bytes;
+        buffer->f(item, buffer->d);
+        item = next;
+    } // while
+    *ptr = '\0';
+
+    assert(ptr == (retval + buffer->total_bytes));
+
+    buffer->head = buffer->tail = NULL;
+    buffer->total_bytes = 0;
+
+    return retval;
+} // buffer_flatten
+
+char *buffer_merge(Buffer **buffers, const size_t n, size_t *_len)
+{
+    Buffer *first = NULL;
+    size_t len = 0;
+    size_t i;
+    for (i = 0; i < n; i++)
+    {
+        Buffer *buffer = buffers[i];
+        if (buffer == NULL)
+            continue;
+        if (first == NULL)
+            first = buffer;
+        len += buffer->total_bytes;
+    } // for
+
+    char *retval = (char *) first ? first->m(len + 1, first->d) : NULL;
+    if (retval == NULL)
+    {
+        *_len = 0;
+        return NULL;
+    } // if
+
+    *_len = len;
+    char *ptr = retval;
+    for (i = 0; i < n; i++)
+    {
+        Buffer *buffer = buffers[i];
+        if (buffer == NULL)
+            continue;
+        BufferBlock *item = buffer->head;
+        while (item != NULL)
+        {
+            BufferBlock *next = item->next;
+            memcpy(ptr, item->data, item->bytes);
+            ptr += item->bytes;
+            buffer->f(item, buffer->d);
+            item = next;
+        } // while
+
+        buffer->head = buffer->tail = NULL;
+        buffer->total_bytes = 0;
+    } // for
+    *ptr = '\0';
+
+    assert(ptr == (retval + len));
+
+    return retval;
+} // buffer_merge
+
+void buffer_destroy(Buffer *buffer)
+{
+    if (buffer != NULL)
+    {
+        MOJOSHADER_free f = buffer->f;
+        void *d = buffer->d;
+        buffer_empty(buffer);
+        f(buffer, d);
+    } // if
+} // buffer_destroy
+
 // end of mojoshader_common.c ...
 
--- a/mojoshader_internal.h	Mon Nov 08 19:06:56 2010 -0500
+++ b/mojoshader_internal.h	Tue Nov 09 05:00:03 2010 -0500
@@ -84,7 +84,7 @@
 #ifdef _MSC_VER
 #include <malloc.h>
 #define va_copy(a, b) a = b
-#define snprintf _snprintf
+#define snprintf _snprintf  // !!! FIXME: not a safe replacement!
 #define strcasecmp stricmp
 typedef unsigned __int8 uint8;
 typedef unsigned __int16 uint16;
@@ -223,6 +223,22 @@
 void errorlist_destroy(ErrorList *list);
 
 
+
+// Dynamic buffers...
+
+typedef struct Buffer Buffer;
+Buffer *buffer_create(size_t blksz,MOJOSHADER_malloc m,MOJOSHADER_free f,void *d);
+int buffer_append(Buffer *buffer, const char *data, size_t len);
+int buffer_append_fmt(Buffer *buffer, const char *fmt, ...) ISPRINTF(2,3);
+int buffer_append_va(Buffer *buffer, const char *fmt, va_list va);
+size_t buffer_size(Buffer *buffer);
+void buffer_empty(Buffer *buffer);
+char *buffer_flatten(Buffer *buffer);
+char *buffer_merge(Buffer **buffers, const size_t n, size_t *_len);
+void buffer_destroy(Buffer *buffer);
+
+
+
 // This is the ID for a D3DXSHADER_CONSTANTTABLE in the bytecode comments.
 #define CTAB_ID 0x42415443  // 0x42415443 == 'CTAB'
 #define CTAB_SIZE 28  // sizeof (D3DXSHADER_CONSTANTTABLE).
--- a/mojoshader_preprocessor.c	Mon Nov 08 19:06:56 2010 -0500
+++ b/mojoshader_preprocessor.c	Tue Nov 09 05:00:03 2010 -0500
@@ -81,6 +81,16 @@
     ctx->free(ptr, ctx->malloc_data);
 } // Free
 
+static void *MallocBridge(int bytes, void *data)
+{
+    return Malloc((Context *) data, (size_t) bytes);
+} // MallocBridge
+
+static void FreeBridge(void *ptr, void *data)
+{
+    Free((Context *) data, ptr);
+} // FreeBridge
+
 static inline char *StrDup(Context *ctx, const char *str)
 {
     char *retval = (char *) Malloc(ctx, strlen(str) + 1);
@@ -284,97 +294,6 @@
 #endif  // !MOJOSHADER_FORCE_INCLUDE_CALLBACKS
 
 
-// !!! FIXME: move this stuff to mojoshader_common.c ...
-// data buffer stuff...
-
-#define BUFFER_LEN (64 * 1024)
-typedef struct BufferList
-{
-    char buffer[BUFFER_LEN];
-    size_t bytes;
-    struct BufferList *next;
-} BufferList;
-
-typedef struct Buffer
-{
-    size_t total_bytes;
-    BufferList head;
-    BufferList *tail;
-} Buffer;
-
-static void init_buffer(Buffer *buffer)
-{
-    buffer->total_bytes = 0;
-    buffer->head.bytes = 0;
-    buffer->head.next = NULL;
-    buffer->tail = &buffer->head;
-} // init_buffer
-
-
-static int add_to_buffer(Buffer *buffer, const char *data,
-                         size_t len, MOJOSHADER_malloc m, void *d)
-{
-    buffer->total_bytes += len;
-    while (len > 0)
-    {
-        const size_t avail = BUFFER_LEN - buffer->tail->bytes;
-        const size_t cpy = (avail > len) ? len : avail;
-        memcpy(buffer->tail->buffer + buffer->tail->bytes, data, cpy);
-        len -= cpy;
-        data += cpy;
-        buffer->tail->bytes += cpy;
-        assert(buffer->tail->bytes <= BUFFER_LEN);
-        if (buffer->tail->bytes == BUFFER_LEN)
-        {
-            BufferList *item = (BufferList *) m(sizeof (BufferList), d);
-            if (item == NULL)
-                return 0;
-            item->bytes = 0;
-            item->next = NULL;
-            buffer->tail->next = item;
-            buffer->tail = item;
-        } // if
-    } // while
-
-    return 1;
-} // add_to_buffer
-
-
-static char *flatten_buffer(Buffer *buffer, MOJOSHADER_malloc m, void *d)
-{
-    char *retval = (char *) m(buffer->total_bytes + 1, d);
-    if (retval == NULL)
-        return NULL;
-    BufferList *item = &buffer->head;
-    char *ptr = retval;
-    while (item != NULL)
-    {
-        BufferList *next = item->next;
-        memcpy(ptr, item->buffer, item->bytes);
-        ptr += item->bytes;
-        item = next;
-    } // while
-    *ptr = '\0';
-
-    assert(ptr == (retval + buffer->total_bytes));
-    return retval;
-} // flatten_buffer
-
-
-static void free_buffer(Buffer *buffer, MOJOSHADER_free f, void *d)
-{
-    // head is statically allocated, so start with head.next...
-    BufferList *item = buffer->head.next;
-    while (item != NULL)
-    {
-        BufferList *next = item->next;
-        f(item, d);
-        item = next;
-    } // while
-    init_buffer(buffer);
-} // free_buffer
-
-
 // !!! FIXME: maybe use these pool magic elsewhere?
 // !!! FIXME: maybe just get rid of this? (maybe the fragmentation isn't a big deal?)
 
@@ -1060,8 +979,6 @@
 
     int params = 0;
     char **idents = NULL;
-    MOJOSHADER_malloc m = ctx->malloc;
-    void *d = ctx->malloc_data;
     static const char space = ' ';
 
     if (state->tokenval == ((Token) ' '))
@@ -1133,8 +1050,7 @@
 
     pushback(state);
 
-    Buffer buffer;
-    init_buffer(&buffer);
+    Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
 
     state->report_whitespace = 1;
     while ((!done) && (!ctx->out_of_memory))
@@ -1153,28 +1069,22 @@
                 break;
 
             case ((Token) ' '):  // may not actually point to ' '.
-                assert(buffer.total_bytes > 0);
-                if (!add_to_buffer(&buffer, &space, 1, m, d))
-                    out_of_memory(ctx);
+                assert(buffer_size(buffer) > 0);
+                buffer_append(buffer, &space, 1);
                 break;
 
             default:
-                if (!add_to_buffer(&buffer,state->token,state->tokenlen,m,d))
-                    out_of_memory(ctx);
+                buffer_append(buffer, state->token, state->tokenlen);
                 break;
         } // switch
     } // while
     state->report_whitespace = 0;
 
+    size_t buflen = buffer_size(buffer) + 1;
     if (!ctx->out_of_memory)
-    {
-        definition = flatten_buffer(&buffer, m, d);
-        if (definition == NULL)
-            out_of_memory(ctx);
-    } // if
+        definition = buffer_flatten(buffer);
 
-    size_t buflen = buffer.total_bytes + 1;
-    free_buffer(&buffer, ctx->free, d);
+    buffer_destroy(buffer);
 
     if (ctx->out_of_memory)
         goto handle_pp_define_failed;
@@ -1326,19 +1236,20 @@
                                   const Define *params)
 {
     char *final = NULL;
-    MOJOSHADER_malloc m = ctx->malloc;
-    MOJOSHADER_free f = ctx->free;
-    void *d = ctx->malloc_data;
 
     // We push the #define and lex it, building a buffer with argument
     //  replacement, stringification, and concatenation.
+    Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
+    if (buffer == NULL)
+        return 0;
+
     IncludeState *state = ctx->include_stack;
     if (!push_source(ctx, state->filename, def->definition,
                      strlen(def->definition), state->line, NULL))
+    {
+        buffer_destroy(buffer);
         return 0;
-
-    Buffer buffer;
-    init_buffer(&buffer);
+    } // if
 
     state = ctx->include_stack;
     while (lexer(state) != TOKEN_EOI)
@@ -1355,9 +1266,9 @@
         } // if
         else
         {
-            if (buffer.total_bytes > 0)
+            if (buffer_size(buffer) > 0)
             {
-                if (!add_to_buffer(&buffer, " ", 1, m, d))
+                if (!buffer_append(buffer, " ", 1))
                     goto replace_and_push_macro_failed;
             } // if
         } // else
@@ -1370,7 +1281,7 @@
             lexer(state);
             assert(state->tokenval != TOKEN_EOI);  // we checked for this.
 
-            if (!add_to_buffer(&buffer, "\"", 1, m, d))
+            if (!buffer_append(buffer, "\"", 1))
                 goto replace_and_push_macro_failed;
 
             if (state->tokenval == TOKEN_IDENTIFIER)
@@ -1383,10 +1294,10 @@
                 } // if
             } // if
 
-            if (!add_to_buffer(&buffer, data, len, m, d))
+            if (!buffer_append(buffer, data, len))
                 goto replace_and_push_macro_failed;
 
-            if (!add_to_buffer(&buffer, "\"", 1, m, d))
+            if (!buffer_append(buffer, "\"", 1))
                 goto replace_and_push_macro_failed;
 
             continue;
@@ -1407,18 +1318,15 @@
             } // if
         } // if
 
-        if (!add_to_buffer(&buffer, data, len, m, d))
+        if (!buffer_append(buffer, data, len))
             goto replace_and_push_macro_failed;
     } // while
 
-    final = flatten_buffer(&buffer, m, d);
+    final = buffer_flatten(buffer);
     if (!final)
-    {
-        out_of_memory(ctx);
         goto replace_and_push_macro_failed;
-    } // if
 
-    free_buffer(&buffer, f, d);
+    buffer_destroy(buffer);
     pop_source(ctx);  // ditch the macro.
     state = ctx->include_stack;
     if (!push_source(ctx, state->filename, final, strlen(final), state->line,
@@ -1432,7 +1340,7 @@
 
 replace_and_push_macro_failed:
     pop_source(ctx);
-    free_buffer(&buffer, f, d);
+    buffer_destroy(buffer);
     return 0;
 } // replace_and_push_macro
 
@@ -1440,9 +1348,6 @@
 static int handle_macro_args(Context *ctx, const char *sym, const Define *def)
 {
     int retval = 0;
-    MOJOSHADER_malloc m = ctx->malloc;
-    MOJOSHADER_free f = ctx->free;
-    void *d = ctx->malloc_data;
     IncludeState *state = ctx->include_stack;
     Define *params = NULL;
     const int expected = (def->paramcount < 0) ? 0 : def->paramcount;
@@ -1461,10 +1366,8 @@
     int paren = 1;
     while (paren > 0)
     {
-        Buffer buffer;
-        Buffer origbuffer;
-        init_buffer(&buffer);
-        init_buffer(&origbuffer);
+        Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
+        Buffer *origbuffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
 
         Token t = lexer(state);
 
@@ -1498,7 +1401,7 @@
                 // don't add whitespace to the start, so we recognize
                 //  void calls correctly.
                 origexpr = expr = " ";
-                origexprlen = (buffer.total_bytes == 0) ? 0 : 1;
+                origexprlen = (buffer_size(buffer) == 0) ? 0 : 1;
             } // else if
 
             else if (t == TOKEN_IDENTIFIER)
@@ -1521,42 +1424,36 @@
 
             assert(expr != NULL);
 
-            if (!add_to_buffer(&buffer, expr, exprlen, m, d))
-            {
-                out_of_memory(ctx);
+            if (!buffer_append(buffer, expr, exprlen))
                 goto handle_macro_args_failed;
-            } // if
 
-            if (!add_to_buffer(&origbuffer, origexpr, origexprlen, m, d))
-            {
-                out_of_memory(ctx);
+            if (!buffer_append(origbuffer, origexpr, origexprlen))
                 goto handle_macro_args_failed;
-            } // if
 
             t = lexer(state);
         } // while
 
-        if (buffer.total_bytes == 0)
+        if (buffer_size(buffer) == 0)
             void_call = ((saw_params == 0) && (paren == 0));
 
         if (saw_params < expected)
         {
-            char *origdefinition = flatten_buffer(&origbuffer, m, d);
-            char *definition = flatten_buffer(&buffer, m, d);
+            char *origdefinition = buffer_flatten(origbuffer);
+            char *definition = buffer_flatten(buffer);
             Define *p = get_define(ctx);
             if ((!origdefinition) || (!definition) || (!p))
             {
                 Free(ctx, origdefinition);
                 Free(ctx, definition);
-                free_buffer(&origbuffer, f, d);
-                free_buffer(&buffer, f, d);
+                buffer_destroy(origbuffer);
+                buffer_destroy(buffer);
                 free_define(ctx, p);
                 goto handle_macro_args_failed;
             } // if
 
             // trim any whitespace from the end of the string...
             int i;
-            for (i = (int) buffer.total_bytes - 1; i >= 0; i--)
+            for (i = (int) buffer_size(buffer) - 1; i >= 0; i--)
             {
                 if (definition[i] == ' ')
                     definition[i] = '\0';
@@ -1564,7 +1461,7 @@
                     break;
             } // for
 
-            for (i = (int) origbuffer.total_bytes - 1; i >= 0; i--)
+            for (i = (int) buffer_size(origbuffer) - 1; i >= 0; i--)
             {
                 if (origdefinition[i] == ' ')
                     origdefinition[i] = '\0';
@@ -1579,8 +1476,8 @@
             params = p;
         } // if
 
-        free_buffer(&buffer, f, d);
-        free_buffer(&origbuffer, f, d);
+        buffer_destroy(buffer);
+        buffer_destroy(origbuffer);
         saw_params++;
     } // while
 
@@ -2263,24 +2160,22 @@
 } // preprocessor_sourcepos
 
 
-static int indent_buffer(Buffer *buffer, int n, int newline,
-                         MOJOSHADER_malloc m, void *d)
+static void indent_buffer(Buffer *buffer, int n, const int newline)
 {
     static char spaces[4] = { ' ', ' ', ' ', ' ' };
     if (newline)
     {
         while (n--)
         {
-            if (!add_to_buffer(buffer, spaces, sizeof (spaces), m, d))
-                return 0;
+            if (!buffer_append(buffer, spaces, sizeof (spaces)))
+                return;
         } // while
     } // if
     else
     {
-        if (!add_to_buffer(buffer, spaces, 1, m, d))
-            return 0;
+        if (!buffer_append(buffer, spaces, 1))
+            return;
     } // else
-    return 1;
 } // indent_buffer
 
 
@@ -2299,6 +2194,7 @@
                              MOJOSHADER_includeClose include_close,
                              MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
 {
+    // !!! FIXME: what's wrong with ENDLINE_STR?
     #ifdef _WINDOWS
     static const char endline[] = { '\r', '\n' };
     #else
@@ -2310,38 +2206,47 @@
     if (!include_open) include_open = MOJOSHADER_internal_include_open;
     if (!include_close) include_close = MOJOSHADER_internal_include_close;
 
-    ErrorList *errors = errorlist_create(m, f, d);
-    if (errors == NULL)
-        return &out_of_mem_data_preprocessor;
-
     Preprocessor *pp = preprocessor_start(filename, source, sourcelen,
                                           include_open, include_close,
                                           defines, define_count, 0, m, f, d);
 
     if (pp == NULL)
+        return &out_of_mem_data_preprocessor;
+
+    ErrorList *errors = errorlist_create(MallocBridge, FreeBridge, pp);
+    if (errors == NULL)
+    {
+        preprocessor_end(pp);
+        return &out_of_mem_data_preprocessor;
+    } // if
+
+    Buffer *buffer = buffer_create(4096, MallocBridge, FreeBridge, pp);
+    if (!buffer)
     {
         errorlist_destroy(errors);
+        preprocessor_end(pp);
         return &out_of_mem_data_preprocessor;
     } // if
 
     Token token = TOKEN_UNKNOWN;
     const char *tokstr = NULL;
 
-    Buffer buffer;
-    init_buffer(&buffer);
-
     int nl = 1;
     int indent = 0;
     unsigned int len = 0;
-    int out_of_memory = 0;
     while ((tokstr = preprocessor_nexttoken(pp, &len, &token)) != NULL)
     {
         int isnewline = 0;
 
         assert(token != TOKEN_EOI);
 
-        if (!out_of_memory)
-            out_of_memory = preprocessor_outofmemory(pp);
+        if (preprocessor_outofmemory(pp))
+        {
+            preprocessor_end(pp);
+            buffer_destroy(buffer);
+            errorlist_destroy(errors);
+            return &out_of_mem_data_preprocessor;
+        } // if
 
         // Microsoft's preprocessor is weird.
         // It ignores newlines, and then inserts its own around certain
@@ -2349,83 +2254,55 @@
         //  be mostly readable, instead of a stream of tokens.
         if ( (token == ((Token) '}')) || (token == ((Token) ';')) )
         {
-            if (!out_of_memory)
-            {
-                if ( (token == ((Token) '}')) && (indent > 0) )
-                    indent--;
+            if ( (token == ((Token) '}')) && (indent > 0) )
+                indent--;
 
-                out_of_memory =
-                    (!indent_buffer(&buffer, indent, nl, m, d)) ||
-                    (!add_to_buffer(&buffer, tokstr, len, m, d)) ||
-                    (!add_to_buffer(&buffer, endline, sizeof (endline), m, d));
+            indent_buffer(buffer, indent, nl);
+            buffer_append(buffer, tokstr, len);
+            buffer_append(buffer, endline, sizeof (endline));
 
-                isnewline = 1;
-            } // if
+            isnewline = 1;
         } // if
 
         else if (token == ((Token) '\n'))
         {
-            if (!out_of_memory)
-            {
-                out_of_memory =
-                    (!add_to_buffer(&buffer, endline, sizeof (endline), m, d));
-            } // if
+            buffer_append(buffer, endline, sizeof (endline));
             isnewline = 1;
         } // else if
 
         else if (token == ((Token) '{'))
         {
-            if (!out_of_memory)
-            {
-                out_of_memory =
-                    (!add_to_buffer(&buffer,endline,sizeof (endline),m,d)) ||
-                    (!indent_buffer(&buffer, indent, 1, m, d)) ||
-                    (!add_to_buffer(&buffer, "{", 1, m, d)) ||
-                    (!add_to_buffer(&buffer,endline,sizeof (endline),m,d));
-                indent++;
-                isnewline = 1;
-            } // if
+            buffer_append(buffer, endline, sizeof (endline));
+            indent_buffer(buffer, indent, 1);
+            buffer_append(buffer, "{", 1);
+            buffer_append(buffer, endline, sizeof (endline));
+            indent++;
+            isnewline = 1;
         } // else if
 
         else if (token == TOKEN_PREPROCESSING_ERROR)
         {
-            if (!out_of_memory)
-            {
-                unsigned int pos = 0;
-                const char *fname = preprocessor_sourcepos(pp, &pos);
-                if (!errorlist_add(errors, fname, (int) pos, tokstr))
-                    out_of_memory = 1;
-            } // if
+            unsigned int pos = 0;
+            const char *fname = preprocessor_sourcepos(pp, &pos);
+            errorlist_add(errors, fname, (int) pos, tokstr);
         } // else if
 
         else
         {
-            if (!out_of_memory)
-            {
-                out_of_memory = (!indent_buffer(&buffer, indent, nl, m, d)) ||
-                                (!add_to_buffer(&buffer, tokstr, len, m, d));
-
-            } // if
+            indent_buffer(buffer, indent, nl);
+            buffer_append(buffer, tokstr, len);
         } // else
 
         nl = isnewline;
-
-        if (out_of_memory)
-        {
-            preprocessor_end(pp);
-            free_buffer(&buffer, f, d);
-            errorlist_destroy(errors);
-            return &out_of_mem_data_preprocessor;
-        } // if
     } // while
     
-    assert((token == TOKEN_EOI) || (out_of_memory));
+    assert(token == TOKEN_EOI);
 
     preprocessor_end(pp);
 
-    const size_t total_bytes = buffer.total_bytes;
-    char *output = flatten_buffer(&buffer, m, d);
-    free_buffer(&buffer, f, d);
+    const size_t total_bytes = buffer_size(buffer);
+    char *output = buffer_flatten(buffer);
+    buffer_destroy(buffer);
     if (output == NULL)
     {
         errorlist_destroy(errors);