Initial work on preprocessor. Not yet complete!
authorRyan C. Gordon <icculus@icculus.org>
Mon, 09 Feb 2009 17:53:54 -0500
changeset 555 940821555fda
parent 554 42dd28107cd8
child 556 04282775cc2c
Initial work on preprocessor. Not yet complete!
CMakeLists.txt
mojoshader.h
mojoshader_internal.h
mojoshader_lexer.c
mojoshader_lexer.re
mojoshader_preprocessor.c
preprocess.c
--- a/CMakeLists.txt	Sat Feb 07 00:54:27 2009 -0500
+++ b/CMakeLists.txt	Mon Feb 09 17:53:54 2009 -0500
@@ -42,6 +42,8 @@
 
 ADD_LIBRARY(mojoshader STATIC
     mojoshader.c
+    mojoshader_preprocessor.c
+    mojoshader_lexer.c
     mojoshader_assembler.c
     mojoshader_opengl.c
 )
@@ -66,6 +68,8 @@
 TARGET_LINK_LIBRARIES(finderrors mojoshader ${SDL_LIBRARY})
 ADD_EXECUTABLE(assemble assemble.c)
 TARGET_LINK_LIBRARIES(assemble mojoshader)
+ADD_EXECUTABLE(preprocess preprocess.c)
+TARGET_LINK_LIBRARIES(preprocess mojoshader)
 
 # End of CMakeLists.txt ...
 
--- a/mojoshader.h	Sat Feb 07 00:54:27 2009 -0500
+++ b/mojoshader.h	Mon Feb 09 17:53:54 2009 -0500
@@ -610,6 +610,198 @@
 
 
 
+/* Preprocessor interface... */
+
+/*
+ * Structure used to pass predefined macros. Maps to D3DXMACRO.
+ */
+typedef struct MOJOSHADER_preprocessorDefine
+{
+    const char *identifier;
+    const char *definition;
+} MOJOSHADER_preprocessorDefine;
+
+/*
+ * Used with the MOJOSHADER_includeOpen callback. Maps to D3DXINCLUDE_TYPE.
+ */
+typedef enum
+{
+    MOJOSHADER_INCLUDETYPE_LOCAL,   /* local header: #include "blah.h" */
+    MOJOSHADER_INCLUDETYPE_SYSTEM   /* system header: #include <blah.h> */
+} MOJOSHADER_includeType;
+
+
+/*
+ * Structure used to return data from preprocessing of a shader...
+ */
+/* !!! FIXME: most of these ints should be unsigned. */
+typedef struct MOJOSHADER_preprocessData
+{
+    /*
+     * The number of elements pointed to by (errors).
+     */
+    int error_count;
+
+    /*
+     * (error_count) elements of data that specify errors that were generated
+     *  by parsing this shader.
+     * This can be NULL if there were no errors or if (error_count) is zero.
+     */
+    MOJOSHADER_error *errors;
+
+    /*
+     * Bytes of output from preprocessing. This is an ASCII string. We
+     *  guarantee it to be NULL-terminated. Will be NULL on error.
+     */
+    const char *output;
+
+    /*
+     * Byte count for output, not counting any null terminator.
+     *  Will be 0 on error.
+     */
+    int output_len;
+
+    /*
+     * This is the malloc implementation you passed to MOJOSHADER_parse().
+     */
+    MOJOSHADER_malloc malloc;
+
+    /*
+     * This is the free implementation you passed to MOJOSHADER_parse().
+     */
+    MOJOSHADER_free free;
+
+    /*
+     * This is the pointer you passed as opaque data for your allocator.
+     */
+    void *malloc_data;
+} MOJOSHADER_preprocessData;
+
+
+/*
+ * This callback allows an app to handle #include statements for the
+ *  preprocessor. When the preprocessor sees an #include, it will call this
+ *  function to obtain the contents of the requested file. This is optional;
+ *  the preprocessor will open files directly if no calback is supplied, but
+ *  this allows an app to retrieve data from something other than the
+ *  traditional filesystem (for example, headers packed in a .zip file or
+ *  headers generated on-the-fly).
+ *
+ * This function maps to ID3DXInclude::Open()
+ *
+ * (inctype) specifies the type of header we wish to include.
+ * (fname) specifies the name of the file specified on the #include line.
+ * (parent) is a string of the entire source file containing the include, in
+ *  its original, not-yet-preprocessed state. Note that this is just the
+ *  contents of the specific file, not all source code that the preprocessor
+ *  has seen through other includes, etc.
+ * (outdata) will be set by the callback to a pointer to the included file's
+ *  contents. The callback is responsible for allocating this however they
+ *  see fit (we provide allocator functions, but you may ignore them). This
+ *  pointer must remain valid until the includeClose callback runs. This
+ *  string does not need to be NULL-terminated.
+ * (outbytes) will be set by the callback to the number of bytes pointed to
+ *  by (outdata).
+ * (m),(f), and (d) are the allocator details that the application passed to
+ *  MojoShader. If these were NULL, MojoShader may have replaced them with its
+ *  own internal allocators.
+ *
+ * The callback returns zero on error, non-zero on success.
+ *
+ * If you supply an includeOpen callback, you must supply includeClose, too.
+ */
+typedef int (*MOJOSHADER_includeOpen)(MOJOSHADER_includeType inctype,
+                            const char *fname, const char *parent,
+                            const char **outdata, unsigned int *outbytes,
+                            MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
+
+/*
+ * This callback allows an app to clean up the results of a previous
+ *  includeOpen callback.
+ *
+ * This function maps to ID3DXInclude::Close()
+ *
+ * (data) is the data that was returned from a previous call to includeOpen.
+ *  It is now safe to deallocate this data.
+ * (m),(f), and (d) are the same allocator details that were passed to your
+ *  includeOpen callback.
+ *
+ * If you supply an includeClose callback, you must supply includeOpen, too.
+ */
+typedef void (*MOJOSHADER_includeClose)(const char *data,
+                            MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
+
+
+/*
+ * This function is optional. Even if you are dealing with shader source
+ *  code, you don't need to explicitly use the preprocessor, as the compiler
+ *  and assembler will use it behind the scenes. In fact, you probably never
+ *  need this function unless you are debugging a custom tool (or debugging
+ *  MojoShader itself).
+ *
+ * Preprocessing roughly follows the syntax of an ANSI C preprocessor, as
+ *  Microsoft's Direct3D assembler and HLSL compiler use this syntax. Please
+ *  note that we try to match the output you'd get from Direct3D's
+ *  preprocessor, which has some quirks if you're expecting output that matches
+ *  a generic C preprocessor.
+ *
+ * This function maps to D3DXPreprocessShader().
+ *
+ * (source) is an ASCII string of text to preprocess. It does not need to be
+ *  NULL-terminated.
+ *
+ * (sourcelen) is the length of the string pointed to by (source), in bytes.
+ *
+ * (defines) points to (define_count) preprocessor definitions, and can be
+ *  NULL. These are treated by the preprocessor as if the source code started
+ *  with one #define for each entry you pass in here.
+ *
+ * (include_open) and (include_close) let the app control the preprocessor's
+ *  behaviour for #include statements. Both are optional and can be NULL, but
+ *  both must be specified if either is specified.
+ *
+ * This will return a MOJOSHADER_preprocessorData. You should pass this
+ *  return value to MOJOSHADER_freePreprocessData() when you are done with
+ *  it.
+ *
+ * This function will never return NULL, even if the system is completely
+ *  out of memory upon entry (in which case, this function returns a static
+ *  MOJOSHADER_preprocessData object, which is still safe to pass to
+ *  MOJOSHADER_freePreprocessData()).
+ *
+ * As preprocessing requires some memory to be allocated, you may provide a
+ *  custom allocator to this function, which will be used to allocate/free
+ *  memory. They function just like malloc() and free(). We do not use
+ *  realloc(). If you don't care, pass NULL in for the allocator functions.
+ *  If your allocator needs instance-specific data, you may supply it with the
+ *  (d) parameter. This pointer is passed as-is to your (m) and (f) functions.
+ *
+ * This function is thread safe, so long as the various callback functions
+ *  are, too, and that the parameters remains intact for the duration of the
+ *  call. This allows you to preprocess several shaders on separate CPU cores
+ *  at the same time.
+ */
+const MOJOSHADER_preprocessData *MOJOSHADER_preprocess(const char *source,
+                             unsigned int sourcelen,
+                             const MOJOSHADER_preprocessorDefine **defines,
+                             unsigned int define_count,
+                             MOJOSHADER_includeOpen include_open,
+                             MOJOSHADER_includeClose include_close,
+                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
+
+
+/*
+ * Call this to dispose of preprocessing results when you are done with them.
+ *  This will call the MOJOSHADER_free function you provided to
+ *  MOJOSHADER_preprocess() multiple times, if you provided one.
+ *  Passing a NULL here is a safe no-op.
+ *
+ * This function is thread safe, so long as any allocator you passed into
+ *  MOJOSHADER_preprocess() is, too.
+ */
+void MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData *data);
+
+
 /* Assembler interface... */
 
 /*
--- a/mojoshader_internal.h	Sat Feb 07 00:54:27 2009 -0500
+++ b/mojoshader_internal.h	Mon Feb 09 17:53:54 2009 -0500
@@ -273,6 +273,81 @@
 } ErrorList;
 
 
+// preprocessor stuff.
+
+typedef enum
+{
+    TOKEN_UNKNOWN = 256,  // start past ASCII character values.
+    TOKEN_IDENTIFIER,
+    TOKEN_INT_LITERAL,
+    TOKEN_FLOAT_LITERAL,
+    TOKEN_STRING_LITERAL,
+    TOKEN_ELLIPSIS,
+    TOKEN_RSHIFT,
+    TOKEN_LSHIFT,
+    TOKEN_ANDAND,
+    TOKEN_OROR,
+    TOKEN_LEQ,
+    TOKEN_GEQ,
+    TOKEN_EQL,
+    TOKEN_NEQ,
+    TOKEN_HASHHASH,
+    TOKEN_PP_INCLUDE,
+    TOKEN_PP_LINE,
+    TOKEN_PP_DEFINE,
+    TOKEN_PP_UNDEF,
+    TOKEN_PP_IF,
+    TOKEN_PP_IFDEF,
+    TOKEN_PP_IFNDEF,
+    TOKEN_PP_ELSE,
+    TOKEN_PP_ELIF,
+    TOKEN_PP_ENDIF,
+    TOKEN_PP_ERROR,
+    TOKEN_PP_INCOMPLETE_COMMENT,
+    TOKEN_EOI
+} Token;
+
+
+// This is opaque.
+struct Preprocessor;
+typedef struct Preprocessor Preprocessor;
+
+typedef struct IncludeState
+{
+    char *filename;
+    int included;
+    Token insert_token;
+    Token insert_token2;
+    char insert_tokchar;
+    const char *source_base;
+    const char *source;
+    const char *token;
+    const unsigned char *lexer_marker;
+    unsigned int bytes_left;
+    unsigned int line;
+    struct IncludeState *next;
+} IncludeState;
+
+Token preprocessor_internal_lexer(IncludeState *s);
+
+// This will only fail if the allocator fails, so it doesn't return any
+//  error code...NULL on failure.
+Preprocessor *preprocessor_start(const char *fname, const char *source,
+                            unsigned int sourcelen,
+                            MOJOSHADER_includeOpen open_callback,
+                            MOJOSHADER_includeClose close_callback,
+                            const MOJOSHADER_preprocessorDefine **defines,
+                            unsigned int define_count,
+                            MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);
+
+void preprocessor_end(Preprocessor *pp);
+const char *preprocessor_error(Preprocessor *pp);
+void preprocessor_clearerror(Preprocessor *pp);
+int preprocessor_outofmemory(Preprocessor *pp);
+const char *preprocessor_nexttoken(Preprocessor *_ctx,
+                                   unsigned int *_len, Token *_token);
+const char *preprocessor_sourcepos(Preprocessor *pp, unsigned int *pos);
+
 
 #endif  // _INCLUDE_MOJOSHADER_INTERNAL_H_
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mojoshader_lexer.c	Mon Feb 09 17:53:54 2009 -0500
@@ -0,0 +1,1309 @@
+/* Generated by re2c 0.13.5 on Mon Feb  9 17:52:01 2009 */
+/**
+ * MojoShader; generate shader programs from bytecode of compiled
+ *  Direct3D shaders.
+ *
+ * Please see the file LICENSE.txt in the source's root directory.
+ *
+ *  This file written by Ryan C. Gordon.
+ */
+
+// This was originally based on examples/pp-c.re from re2c: http://re2c.org/
+//   re2c is public domain code.
+//
+// You build mojoshader_lexer_preprocessor.c from the .re file with re2c...
+// re2c -is -o mojoshader_lexer_preprocessor.c mojoshader_lexer_preprocessor.re
+//
+// Changes to the lexer are done to the .re file, not the C code!
+//
+// Please note that this isn't a perfect C lexer, since it is used for both
+//  HLSL and shader assembly language, and follows the quirks of Microsoft's
+//  tools.
+
+#define __MOJOSHADER_INTERNAL__ 1
+#include "mojoshader_internal.h"
+
+typedef unsigned char uchar;
+
+#define RET(t) { update_state(s, cursor, token); return t; }
+#define YYCTYPE uchar
+#define YYCURSOR cursor
+#define YYLIMIT limit
+#define YYMARKER s->lexer_marker
+#define YYFILL(n) { if ((n) == 1) { RET(TOKEN_EOI); } }
+
+static void update_state(IncludeState *s, const uchar *cur, const uchar *tok)
+{
+    s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
+    s->source = (const char *) cur;
+    s->token = (const char *) tok;
+} // update_state
+
+Token preprocessor_internal_lexer(IncludeState *s)
+{
+    const uchar *cursor = (const uchar *) s->source;
+    const uchar *token;
+    const uchar *limit = cursor + s->bytes_left;
+
+scanner_loop:
+    token = cursor;
+
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_EOI);
+
+
+
+
+{
+	YYCTYPE yych;
+	unsigned int yyaccept = 0;
+
+	if ((YYLIMIT - YYCURSOR) < 8) YYFILL(8);
+	yych = *YYCURSOR;
+	switch (yych) {
+	case '\t':
+	case '\v':
+	case '\f':
+	case ' ':	goto yy62;
+	case '\n':	goto yy64;
+	case '\r':	goto yy66;
+	case '!':	goto yy24;
+	case '"':	goto yy13;
+	case '#':	goto yy26;
+	case '%':	goto yy46;
+	case '&':	goto yy18;
+	case '\'':	goto yy9;
+	case '(':	goto yy28;
+	case ')':	goto yy30;
+	case '*':	goto yy44;
+	case '+':	goto yy42;
+	case ',':	goto yy36;
+	case '-':	goto yy40;
+	case '.':	goto yy11;
+	case '/':	goto yy2;
+	case '0':	goto yy6;
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':	goto yy8;
+	case ':':	goto yy50;
+	case ';':	goto yy52;
+	case '<':	goto yy16;
+	case '=':	goto yy22;
+	case '>':	goto yy14;
+	case '?':	goto yy58;
+	case 'A':
+	case 'B':
+	case 'C':
+	case 'D':
+	case 'E':
+	case 'F':
+	case 'G':
+	case 'H':
+	case 'I':
+	case 'J':
+	case 'K':
+	case 'L':
+	case 'M':
+	case 'N':
+	case 'O':
+	case 'P':
+	case 'Q':
+	case 'R':
+	case 'S':
+	case 'T':
+	case 'U':
+	case 'V':
+	case 'W':
+	case 'X':
+	case 'Y':
+	case 'Z':
+	case '_':
+	case 'a':
+	case 'b':
+	case 'c':
+	case 'd':
+	case 'e':
+	case 'f':
+	case 'g':
+	case 'h':
+	case 'i':
+	case 'j':
+	case 'k':
+	case 'l':
+	case 'm':
+	case 'n':
+	case 'o':
+	case 'p':
+	case 'q':
+	case 'r':
+	case 's':
+	case 't':
+	case 'u':
+	case 'v':
+	case 'w':
+	case 'x':
+	case 'y':
+	case 'z':	goto yy4;
+	case '[':	goto yy32;
+	case '\\':	goto yy60;
+	case ']':	goto yy34;
+	case '^':	goto yy48;
+	case '{':	goto yy54;
+	case '|':	goto yy20;
+	case '}':	goto yy56;
+	case '~':	goto yy38;
+	default:	goto yy67;
+	}
+yy2:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '*') goto yy209;
+	if (yych == '/') goto yy207;
+	{ RET('/'); }
+yy4:
+	++YYCURSOR;
+	yych = *YYCURSOR;
+	goto yy206;
+yy5:
+	{ RET(TOKEN_IDENTIFIER); }
+yy6:
+	yyaccept = 0;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych <= 'X') {
+		if (yych <= 'T') {
+			if (yych == 'L') goto yy181;
+			goto yy197;
+		} else {
+			if (yych <= 'U') goto yy181;
+			if (yych <= 'W') goto yy197;
+			goto yy198;
+		}
+	} else {
+		if (yych <= 't') {
+			if (yych == 'l') goto yy181;
+			goto yy197;
+		} else {
+			if (yych <= 'u') goto yy181;
+			if (yych == 'x') goto yy198;
+			goto yy197;
+		}
+	}
+yy7:
+	{ RET(TOKEN_INT_LITERAL); }
+yy8:
+	yyaccept = 0;
+	yych = *(YYMARKER = ++YYCURSOR);
+	goto yy179;
+yy9:
+	yyaccept = 1;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych != '\n') goto yy169;
+yy10:
+	{ printf("bad char\n"); goto scanner_loop; }
+yy11:
+	yyaccept = 2;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych == '.') goto yy157;
+	if (yych <= '/') goto yy12;
+	if (yych <= '9') goto yy158;
+yy12:
+	{ RET('.'); }
+yy13:
+	yyaccept = 1;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych == '\n') goto yy10;
+	goto yy148;
+yy14:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) <= '<') goto yy15;
+	if (yych <= '=') goto yy143;
+	if (yych <= '>') goto yy145;
+yy15:
+	{ RET('>'); }
+yy16:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) <= ';') goto yy17;
+	if (yych <= '<') goto yy141;
+	if (yych <= '=') goto yy139;
+yy17:
+	{ RET('<'); }
+yy18:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '&') goto yy137;
+	{ RET('&'); }
+yy20:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '|') goto yy135;
+	{ RET('|'); }
+yy22:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '=') goto yy133;
+	{ RET('='); }
+yy24:
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '=') goto yy131;
+	{ RET('!'); }
+yy26:
+	yyaccept = 3;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych <= 'c') {
+		if (yych <= 0x1F) {
+			if (yych == '\t') goto yy72;
+		} else {
+			if (yych <= ' ') goto yy72;
+			if (yych == '#') goto yy79;
+		}
+	} else {
+		if (yych <= 'k') {
+			if (yych <= 'e') goto yy72;
+			if (yych == 'i') goto yy72;
+		} else {
+			if (yych <= 'l') goto yy72;
+			if (yych == 'u') goto yy72;
+		}
+	}
+yy27:
+	{ RET('#'); }
+yy28:
+	++YYCURSOR;
+	{ RET('('); }
+yy30:
+	++YYCURSOR;
+	{ RET(')'); }
+yy32:
+	++YYCURSOR;
+	{ RET('['); }
+yy34:
+	++YYCURSOR;
+	{ RET(']'); }
+yy36:
+	++YYCURSOR;
+	{ RET(','); }
+yy38:
+	++YYCURSOR;
+	{ RET('~'); }
+yy40:
+	++YYCURSOR;
+	{ RET('-'); }
+yy42:
+	++YYCURSOR;
+	{ RET('+'); }
+yy44:
+	++YYCURSOR;
+	{ RET('*'); }
+yy46:
+	++YYCURSOR;
+	{ RET('%'); }
+yy48:
+	++YYCURSOR;
+	{ RET('^'); }
+yy50:
+	++YYCURSOR;
+	{ RET(':'); }
+yy52:
+	++YYCURSOR;
+	{ RET(';'); }
+yy54:
+	++YYCURSOR;
+	{ RET('{'); }
+yy56:
+	++YYCURSOR;
+	{ RET('}'); }
+yy58:
+	++YYCURSOR;
+	{ RET('?'); }
+yy60:
+	++YYCURSOR;
+	{ RET('\\'); }
+yy62:
+	++YYCURSOR;
+	yych = *YYCURSOR;
+	goto yy70;
+yy63:
+	{ goto scanner_loop; }
+yy64:
+	++YYCURSOR;
+yy65:
+	{ s->line++; goto scanner_loop; }
+yy66:
+	yych = *++YYCURSOR;
+	if (yych == '\n') goto yy68;
+	goto yy65;
+yy67:
+	yych = *++YYCURSOR;
+	goto yy10;
+yy68:
+	yych = *++YYCURSOR;
+	goto yy65;
+yy69:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy70:
+	if (yych <= '\n') {
+		if (yych == '\t') goto yy69;
+		goto yy63;
+	} else {
+		if (yych <= '\f') goto yy69;
+		if (yych == ' ') goto yy69;
+		goto yy63;
+	}
+yy71:
+	++YYCURSOR;
+	if ((YYLIMIT - YYCURSOR) < 7) YYFILL(7);
+	yych = *YYCURSOR;
+yy72:
+	if (yych <= 'e') {
+		if (yych <= 0x1F) {
+			if (yych == '\t') goto yy71;
+		} else {
+			if (yych <= ' ') goto yy71;
+			if (yych <= 'c') goto yy73;
+			if (yych <= 'd') goto yy77;
+			goto yy74;
+		}
+	} else {
+		if (yych <= 'k') {
+			if (yych == 'i') goto yy75;
+		} else {
+			if (yych <= 'l') goto yy78;
+			if (yych == 'u') goto yy76;
+		}
+	}
+yy73:
+	YYCURSOR = YYMARKER;
+	if (yyaccept <= 2) {
+		if (yyaccept <= 1) {
+			if (yyaccept <= 0) {
+				goto yy7;
+			} else {
+				goto yy10;
+			}
+		} else {
+			goto yy12;
+		}
+	} else {
+		if (yyaccept <= 4) {
+			if (yyaccept <= 3) {
+				goto yy27;
+			} else {
+				goto yy98;
+			}
+		} else {
+			goto yy160;
+		}
+	}
+yy74:
+	yych = *++YYCURSOR;
+	if (yych <= 'm') {
+		if (yych == 'l') goto yy114;
+		goto yy73;
+	} else {
+		if (yych <= 'n') goto yy115;
+		if (yych == 'r') goto yy116;
+		goto yy73;
+	}
+yy75:
+	yych = *++YYCURSOR;
+	if (yych == 'f') goto yy97;
+	if (yych == 'n') goto yy96;
+	goto yy73;
+yy76:
+	yych = *++YYCURSOR;
+	if (yych == 'n') goto yy91;
+	goto yy73;
+yy77:
+	yych = *++YYCURSOR;
+	if (yych == 'e') goto yy85;
+	goto yy73;
+yy78:
+	yych = *++YYCURSOR;
+	if (yych == 'i') goto yy81;
+	goto yy73;
+yy79:
+	++YYCURSOR;
+	{ RET(TOKEN_HASHHASH); }
+yy81:
+	yych = *++YYCURSOR;
+	if (yych != 'n') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_LINE); }
+yy85:
+	yych = *++YYCURSOR;
+	if (yych != 'f') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'i') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'n') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_DEFINE); }
+yy91:
+	yych = *++YYCURSOR;
+	if (yych != 'd') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'f') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_UNDEF); }
+yy96:
+	yych = *++YYCURSOR;
+	if (yych == 'c') goto yy108;
+	goto yy73;
+yy97:
+	yyaccept = 4;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych == 'd') goto yy100;
+	if (yych == 'n') goto yy99;
+yy98:
+	{ RET(TOKEN_PP_IF); }
+yy99:
+	yych = *++YYCURSOR;
+	if (yych == 'd') goto yy104;
+	goto yy73;
+yy100:
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'f') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_IFDEF); }
+yy104:
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'f') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_IFNDEF); }
+yy108:
+	yych = *++YYCURSOR;
+	if (yych != 'l') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'u') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'd') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_INCLUDE); }
+yy114:
+	yych = *++YYCURSOR;
+	if (yych == 'i') goto yy125;
+	if (yych == 's') goto yy126;
+	goto yy73;
+yy115:
+	yych = *++YYCURSOR;
+	if (yych == 'd') goto yy121;
+	goto yy73;
+yy116:
+	yych = *++YYCURSOR;
+	if (yych != 'r') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'o') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'r') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_ERROR); }
+yy121:
+	yych = *++YYCURSOR;
+	if (yych != 'i') goto yy73;
+	yych = *++YYCURSOR;
+	if (yych != 'f') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_ENDIF); }
+yy125:
+	yych = *++YYCURSOR;
+	if (yych == 'f') goto yy129;
+	goto yy73;
+yy126:
+	yych = *++YYCURSOR;
+	if (yych != 'e') goto yy73;
+	++YYCURSOR;
+	{ RET(TOKEN_PP_ELSE); }
+yy129:
+	++YYCURSOR;
+	{ RET(TOKEN_PP_ELIF); }
+yy131:
+	++YYCURSOR;
+	{ RET(TOKEN_NEQ); }
+yy133:
+	++YYCURSOR;
+	{ RET(TOKEN_EQL); }
+yy135:
+	++YYCURSOR;
+	{ RET(TOKEN_OROR); }
+yy137:
+	++YYCURSOR;
+	{ RET(TOKEN_ANDAND); }
+yy139:
+	++YYCURSOR;
+	{ RET(TOKEN_LEQ); }
+yy141:
+	++YYCURSOR;
+	{ RET(TOKEN_LSHIFT); }
+yy143:
+	++YYCURSOR;
+	{ RET(TOKEN_GEQ); }
+yy145:
+	++YYCURSOR;
+	{ RET(TOKEN_RSHIFT); }
+yy147:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy148:
+	if (yych <= '!') {
+		if (yych == '\n') goto yy73;
+		goto yy147;
+	} else {
+		if (yych <= '"') goto yy150;
+		if (yych != '\\') goto yy147;
+	}
+yy149:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'b') {
+		if (yych <= '7') {
+			if (yych <= '&') {
+				if (yych == '"') goto yy147;
+				goto yy73;
+			} else {
+				if (yych <= '\'') goto yy147;
+				if (yych <= '/') goto yy73;
+				goto yy153;
+			}
+		} else {
+			if (yych <= '[') {
+				if (yych == '?') goto yy147;
+				goto yy73;
+			} else {
+				if (yych <= '\\') goto yy147;
+				if (yych <= '`') goto yy73;
+				goto yy147;
+			}
+		}
+	} else {
+		if (yych <= 'r') {
+			if (yych <= 'm') {
+				if (yych == 'f') goto yy147;
+				goto yy73;
+			} else {
+				if (yych <= 'n') goto yy147;
+				if (yych <= 'q') goto yy73;
+				goto yy147;
+			}
+		} else {
+			if (yych <= 'u') {
+				if (yych == 't') goto yy147;
+				goto yy73;
+			} else {
+				if (yych <= 'v') goto yy147;
+				if (yych == 'x') goto yy152;
+				goto yy73;
+			}
+		}
+	}
+yy150:
+	++YYCURSOR;
+	{ RET(TOKEN_STRING_LITERAL); }
+yy152:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '@') {
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy155;
+		goto yy73;
+	} else {
+		if (yych <= 'F') goto yy155;
+		if (yych <= '`') goto yy73;
+		if (yych <= 'f') goto yy155;
+		goto yy73;
+	}
+yy153:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '"') {
+		if (yych == '\n') goto yy73;
+		if (yych <= '!') goto yy147;
+		goto yy150;
+	} else {
+		if (yych <= '7') {
+			if (yych <= '/') goto yy147;
+			goto yy153;
+		} else {
+			if (yych == '\\') goto yy149;
+			goto yy147;
+		}
+	}
+yy155:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '9') {
+		if (yych <= '!') {
+			if (yych == '\n') goto yy73;
+			goto yy147;
+		} else {
+			if (yych <= '"') goto yy150;
+			if (yych <= '/') goto yy147;
+			goto yy155;
+		}
+	} else {
+		if (yych <= '[') {
+			if (yych <= '@') goto yy147;
+			if (yych <= 'F') goto yy155;
+			goto yy147;
+		} else {
+			if (yych <= '\\') goto yy149;
+			if (yych <= '`') goto yy147;
+			if (yych <= 'f') goto yy155;
+			goto yy147;
+		}
+	}
+yy157:
+	yych = *++YYCURSOR;
+	if (yych == '.') goto yy166;
+	goto yy73;
+yy158:
+	yyaccept = 5;
+	YYMARKER = ++YYCURSOR;
+	if ((YYLIMIT - YYCURSOR) < 3) YYFILL(3);
+	yych = *YYCURSOR;
+	if (yych <= 'K') {
+		if (yych <= 'D') {
+			if (yych <= '/') goto yy160;
+			if (yych <= '9') goto yy158;
+		} else {
+			if (yych <= 'E') goto yy161;
+			if (yych <= 'F') goto yy162;
+		}
+	} else {
+		if (yych <= 'e') {
+			if (yych <= 'L') goto yy162;
+			if (yych >= 'e') goto yy161;
+		} else {
+			if (yych <= 'f') goto yy162;
+			if (yych == 'l') goto yy162;
+		}
+	}
+yy160:
+	{ RET(TOKEN_FLOAT_LITERAL); }
+yy161:
+	yych = *++YYCURSOR;
+	if (yych <= ',') {
+		if (yych == '+') goto yy163;
+		goto yy73;
+	} else {
+		if (yych <= '-') goto yy163;
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy164;
+		goto yy73;
+	}
+yy162:
+	yych = *++YYCURSOR;
+	goto yy160;
+yy163:
+	yych = *++YYCURSOR;
+	if (yych <= '/') goto yy73;
+	if (yych >= ':') goto yy73;
+yy164:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'K') {
+		if (yych <= '9') {
+			if (yych <= '/') goto yy160;
+			goto yy164;
+		} else {
+			if (yych == 'F') goto yy162;
+			goto yy160;
+		}
+	} else {
+		if (yych <= 'f') {
+			if (yych <= 'L') goto yy162;
+			if (yych <= 'e') goto yy160;
+			goto yy162;
+		} else {
+			if (yych == 'l') goto yy162;
+			goto yy160;
+		}
+	}
+yy166:
+	++YYCURSOR;
+	{ RET(TOKEN_ELLIPSIS); }
+yy168:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy169:
+	if (yych <= '&') {
+		if (yych == '\n') goto yy73;
+		goto yy168;
+	} else {
+		if (yych <= '\'') goto yy171;
+		if (yych != '\\') goto yy168;
+	}
+yy170:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'b') {
+		if (yych <= '7') {
+			if (yych <= '&') {
+				if (yych == '"') goto yy168;
+				goto yy73;
+			} else {
+				if (yych <= '\'') goto yy168;
+				if (yych <= '/') goto yy73;
+				goto yy173;
+			}
+		} else {
+			if (yych <= '[') {
+				if (yych == '?') goto yy168;
+				goto yy73;
+			} else {
+				if (yych <= '\\') goto yy168;
+				if (yych <= '`') goto yy73;
+				goto yy168;
+			}
+		}
+	} else {
+		if (yych <= 'r') {
+			if (yych <= 'm') {
+				if (yych == 'f') goto yy168;
+				goto yy73;
+			} else {
+				if (yych <= 'n') goto yy168;
+				if (yych <= 'q') goto yy73;
+				goto yy168;
+			}
+		} else {
+			if (yych <= 'u') {
+				if (yych == 't') goto yy168;
+				goto yy73;
+			} else {
+				if (yych <= 'v') goto yy168;
+				if (yych == 'x') goto yy172;
+				goto yy73;
+			}
+		}
+	}
+yy171:
+	yych = *++YYCURSOR;
+	goto yy7;
+yy172:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '@') {
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy175;
+		goto yy73;
+	} else {
+		if (yych <= 'F') goto yy175;
+		if (yych <= '`') goto yy73;
+		if (yych <= 'f') goto yy175;
+		goto yy73;
+	}
+yy173:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '\'') {
+		if (yych == '\n') goto yy73;
+		if (yych <= '&') goto yy168;
+		goto yy171;
+	} else {
+		if (yych <= '7') {
+			if (yych <= '/') goto yy168;
+			goto yy173;
+		} else {
+			if (yych == '\\') goto yy170;
+			goto yy168;
+		}
+	}
+yy175:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= '9') {
+		if (yych <= '&') {
+			if (yych == '\n') goto yy73;
+			goto yy168;
+		} else {
+			if (yych <= '\'') goto yy171;
+			if (yych <= '/') goto yy168;
+			goto yy175;
+		}
+	} else {
+		if (yych <= '[') {
+			if (yych <= '@') goto yy168;
+			if (yych <= 'F') goto yy175;
+			goto yy168;
+		} else {
+			if (yych <= '\\') goto yy170;
+			if (yych <= '`') goto yy168;
+			if (yych <= 'f') goto yy175;
+			goto yy168;
+		}
+	}
+yy177:
+	yyaccept = 5;
+	yych = *(YYMARKER = ++YYCURSOR);
+	if (yych == 'E') goto yy188;
+	if (yych == 'e') goto yy188;
+	goto yy187;
+yy178:
+	yyaccept = 0;
+	YYMARKER = ++YYCURSOR;
+	if ((YYLIMIT - YYCURSOR) < 4) YYFILL(4);
+	yych = *YYCURSOR;
+yy179:
+	if (yych <= 'L') {
+		if (yych <= '9') {
+			if (yych == '.') goto yy177;
+			if (yych <= '/') goto yy7;
+			goto yy178;
+		} else {
+			if (yych == 'E') goto yy180;
+			if (yych <= 'K') goto yy7;
+			goto yy181;
+		}
+	} else {
+		if (yych <= 'e') {
+			if (yych == 'U') goto yy181;
+			if (yych <= 'd') goto yy7;
+		} else {
+			if (yych <= 'l') {
+				if (yych <= 'k') goto yy7;
+				goto yy181;
+			} else {
+				if (yych == 'u') goto yy181;
+				goto yy7;
+			}
+		}
+	}
+yy180:
+	yych = *++YYCURSOR;
+	if (yych <= ',') {
+		if (yych == '+') goto yy183;
+		goto yy73;
+	} else {
+		if (yych <= '-') goto yy183;
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy184;
+		goto yy73;
+	}
+yy181:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'U') {
+		if (yych == 'L') goto yy181;
+		if (yych <= 'T') goto yy7;
+		goto yy181;
+	} else {
+		if (yych <= 'l') {
+			if (yych <= 'k') goto yy7;
+			goto yy181;
+		} else {
+			if (yych == 'u') goto yy181;
+			goto yy7;
+		}
+	}
+yy183:
+	yych = *++YYCURSOR;
+	if (yych <= '/') goto yy73;
+	if (yych >= ':') goto yy73;
+yy184:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'K') {
+		if (yych <= '9') {
+			if (yych <= '/') goto yy160;
+			goto yy184;
+		} else {
+			if (yych == 'F') goto yy162;
+			goto yy160;
+		}
+	} else {
+		if (yych <= 'f') {
+			if (yych <= 'L') goto yy162;
+			if (yych <= 'e') goto yy160;
+			goto yy162;
+		} else {
+			if (yych == 'l') goto yy162;
+			goto yy160;
+		}
+	}
+yy186:
+	yyaccept = 5;
+	YYMARKER = ++YYCURSOR;
+	if ((YYLIMIT - YYCURSOR) < 3) YYFILL(3);
+	yych = *YYCURSOR;
+yy187:
+	if (yych <= 'K') {
+		if (yych <= 'D') {
+			if (yych <= '/') goto yy160;
+			if (yych <= '9') goto yy186;
+			goto yy160;
+		} else {
+			if (yych <= 'E') goto yy192;
+			if (yych <= 'F') goto yy162;
+			goto yy160;
+		}
+	} else {
+		if (yych <= 'e') {
+			if (yych <= 'L') goto yy162;
+			if (yych <= 'd') goto yy160;
+			goto yy192;
+		} else {
+			if (yych <= 'f') goto yy162;
+			if (yych == 'l') goto yy162;
+			goto yy160;
+		}
+	}
+yy188:
+	yych = *++YYCURSOR;
+	if (yych <= ',') {
+		if (yych != '+') goto yy73;
+	} else {
+		if (yych <= '-') goto yy189;
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy190;
+		goto yy73;
+	}
+yy189:
+	yych = *++YYCURSOR;
+	if (yych <= '/') goto yy73;
+	if (yych >= ':') goto yy73;
+yy190:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'K') {
+		if (yych <= '9') {
+			if (yych <= '/') goto yy160;
+			goto yy190;
+		} else {
+			if (yych == 'F') goto yy162;
+			goto yy160;
+		}
+	} else {
+		if (yych <= 'f') {
+			if (yych <= 'L') goto yy162;
+			if (yych <= 'e') goto yy160;
+			goto yy162;
+		} else {
+			if (yych == 'l') goto yy162;
+			goto yy160;
+		}
+	}
+yy192:
+	yych = *++YYCURSOR;
+	if (yych <= ',') {
+		if (yych != '+') goto yy73;
+	} else {
+		if (yych <= '-') goto yy193;
+		if (yych <= '/') goto yy73;
+		if (yych <= '9') goto yy194;
+		goto yy73;
+	}
+yy193:
+	yych = *++YYCURSOR;
+	if (yych <= '/') goto yy73;
+	if (yych >= ':') goto yy73;
+yy194:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'K') {
+		if (yych <= '9') {
+			if (yych <= '/') goto yy160;
+			goto yy194;
+		} else {
+			if (yych == 'F') goto yy162;
+			goto yy160;
+		}
+	} else {
+		if (yych <= 'f') {
+			if (yych <= 'L') goto yy162;
+			if (yych <= 'e') goto yy160;
+			goto yy162;
+		} else {
+			if (yych == 'l') goto yy162;
+			goto yy160;
+		}
+	}
+yy196:
+	yyaccept = 0;
+	YYMARKER = ++YYCURSOR;
+	if ((YYLIMIT - YYCURSOR) < 4) YYFILL(4);
+	yych = *YYCURSOR;
+yy197:
+	if (yych <= 'L') {
+		if (yych <= '9') {
+			if (yych == '.') goto yy177;
+			if (yych <= '/') goto yy7;
+			goto yy196;
+		} else {
+			if (yych == 'E') goto yy180;
+			if (yych <= 'K') goto yy7;
+			goto yy203;
+		}
+	} else {
+		if (yych <= 'e') {
+			if (yych == 'U') goto yy203;
+			if (yych <= 'd') goto yy7;
+			goto yy180;
+		} else {
+			if (yych <= 'l') {
+				if (yych <= 'k') goto yy7;
+				goto yy203;
+			} else {
+				if (yych == 'u') goto yy203;
+				goto yy7;
+			}
+		}
+	}
+yy198:
+	yych = *++YYCURSOR;
+	if (yych <= '@') {
+		if (yych <= '/') goto yy73;
+		if (yych >= ':') goto yy73;
+	} else {
+		if (yych <= 'F') goto yy199;
+		if (yych <= '`') goto yy73;
+		if (yych >= 'g') goto yy73;
+	}
+yy199:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'T') {
+		if (yych <= '@') {
+			if (yych <= '/') goto yy7;
+			if (yych <= '9') goto yy199;
+			goto yy7;
+		} else {
+			if (yych <= 'F') goto yy199;
+			if (yych != 'L') goto yy7;
+		}
+	} else {
+		if (yych <= 'k') {
+			if (yych <= 'U') goto yy201;
+			if (yych <= '`') goto yy7;
+			if (yych <= 'f') goto yy199;
+			goto yy7;
+		} else {
+			if (yych <= 'l') goto yy201;
+			if (yych != 'u') goto yy7;
+		}
+	}
+yy201:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'U') {
+		if (yych == 'L') goto yy201;
+		if (yych <= 'T') goto yy7;
+		goto yy201;
+	} else {
+		if (yych <= 'l') {
+			if (yych <= 'k') goto yy7;
+			goto yy201;
+		} else {
+			if (yych == 'u') goto yy201;
+			goto yy7;
+		}
+	}
+yy203:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+	if (yych <= 'U') {
+		if (yych == 'L') goto yy203;
+		if (yych <= 'T') goto yy7;
+		goto yy203;
+	} else {
+		if (yych <= 'l') {
+			if (yych <= 'k') goto yy7;
+			goto yy203;
+		} else {
+			if (yych == 'u') goto yy203;
+			goto yy7;
+		}
+	}
+yy205:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy206:
+	if (yych <= 'Z') {
+		if (yych <= '/') goto yy5;
+		if (yych <= '9') goto yy205;
+		if (yych <= '@') goto yy5;
+		goto yy205;
+	} else {
+		if (yych <= '_') {
+			if (yych <= '^') goto yy5;
+			goto yy205;
+		} else {
+			if (yych <= '`') goto yy5;
+			if (yych <= 'z') goto yy205;
+			goto yy5;
+		}
+	}
+yy207:
+	++YYCURSOR;
+	{ goto singlelinecomment; }
+yy209:
+	++YYCURSOR;
+	{ goto multilinecomment; }
+}
+
+
+multilinecomment:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_PP_INCOMPLETE_COMMENT);
+// The "*\/" is just to avoid screwing up text editor syntax highlighting.
+
+{
+	YYCTYPE yych;
+	if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
+	yych = *YYCURSOR;
+	if (yych <= '\f') {
+		if (yych == '\n') goto yy215;
+		goto yy218;
+	} else {
+		if (yych <= '\r') goto yy217;
+		if (yych != '*') goto yy218;
+	}
+	++YYCURSOR;
+	if ((yych = *YYCURSOR) == '/') goto yy220;
+yy214:
+	{ goto multilinecomment; }
+yy215:
+	++YYCURSOR;
+yy216:
+	{ s->line++; goto multilinecomment; }
+yy217:
+	yych = *++YYCURSOR;
+	if (yych == '\n') goto yy219;
+	goto yy216;
+yy218:
+	yych = *++YYCURSOR;
+	goto yy214;
+yy219:
+	yych = *++YYCURSOR;
+	goto yy216;
+yy220:
+	++YYCURSOR;
+	{ goto scanner_loop; }
+}
+
+
+singlelinecomment:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_EOI);
+
+{
+	YYCTYPE yych;
+	if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
+	yych = *YYCURSOR;
+	if (yych == '\n') goto yy224;
+	if (yych == '\r') goto yy226;
+	goto yy227;
+yy224:
+	++YYCURSOR;
+yy225:
+	{ s->line++; goto scanner_loop; }
+yy226:
+	yych = *++YYCURSOR;
+	if (yych == '\n') goto yy229;
+	goto yy225;
+yy227:
+	++YYCURSOR;
+	{ goto singlelinecomment; }
+yy229:
+	++YYCURSOR;
+	yych = *YYCURSOR;
+	goto yy225;
+}
+
+
+// !!! FIXME
+/*
+bad_chars:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_BAD_TOKEN);
+*/
+
+
+{
+	YYCTYPE yych;
+	if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
+	yych = *YYCURSOR;
+	if (yych <= '\f') {
+		if (yych <= 0x08) goto yy237;
+		if (yych != '\n') goto yy235;
+	} else {
+		if (yych <= '\r') goto yy234;
+		if (yych == ' ') goto yy235;
+		goto yy237;
+	}
+	++YYCURSOR;
+yy233:
+	{ s->line++; goto scanner_loop; }
+yy234:
+	yych = *++YYCURSOR;
+	if (yych == '\n') goto yy241;
+	goto yy233;
+yy235:
+	++YYCURSOR;
+	yych = *YYCURSOR;
+	goto yy240;
+yy236:
+	{ goto scanner_loop; }
+yy237:
+	++YYCURSOR;
+	{ goto singlelinecomment; }
+yy239:
+	++YYCURSOR;
+	if (YYLIMIT <= YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy240:
+	if (yych <= '\n') {
+		if (yych == '\t') goto yy239;
+		goto yy236;
+	} else {
+		if (yych <= '\f') goto yy239;
+		if (yych == ' ') goto yy239;
+		goto yy236;
+	}
+yy241:
+	++YYCURSOR;
+	yych = *YYCURSOR;
+	goto yy233;
+}
+
+
+    assert(0 && "Shouldn't hit this code");
+    RET(TOKEN_UNKNOWN);
+} // preprocessor_internal_lexer
+
+// end of mojoshader_lexer_preprocessor.re (or .c) ...
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mojoshader_lexer.re	Mon Feb 09 17:53:54 2009 -0500
@@ -0,0 +1,174 @@
+/**
+ * MojoShader; generate shader programs from bytecode of compiled
+ *  Direct3D shaders.
+ *
+ * Please see the file LICENSE.txt in the source's root directory.
+ *
+ *  This file written by Ryan C. Gordon.
+ */
+
+// This was originally based on examples/pp-c.re from re2c: http://re2c.org/
+//   re2c is public domain code.
+//
+// You build mojoshader_lexer_preprocessor.c from the .re file with re2c...
+// re2c -is -o mojoshader_lexer_preprocessor.c mojoshader_lexer_preprocessor.re
+//
+// Changes to the lexer are done to the .re file, not the C code!
+//
+// Please note that this isn't a perfect C lexer, since it is used for both
+//  HLSL and shader assembly language, and follows the quirks of Microsoft's
+//  tools.
+
+#define __MOJOSHADER_INTERNAL__ 1
+#include "mojoshader_internal.h"
+
+typedef unsigned char uchar;
+
+#define RET(t) { update_state(s, cursor, token); return t; }
+#define YYCTYPE uchar
+#define YYCURSOR cursor
+#define YYLIMIT limit
+#define YYMARKER s->lexer_marker
+#define YYFILL(n) { if ((n) == 1) { RET(TOKEN_EOI); } }
+
+static void update_state(IncludeState *s, const uchar *cur, const uchar *tok)
+{
+    s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
+    s->source = (const char *) cur;
+    s->token = (const char *) tok;
+} // update_state
+
+Token preprocessor_internal_lexer(IncludeState *s)
+{
+    const uchar *cursor = (const uchar *) s->source;
+    const uchar *token;
+    const uchar *limit = cursor + s->bytes_left;
+
+scanner_loop:
+    token = cursor;
+
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_EOI);
+
+/*!re2c
+    any = [\000-\377];
+    O = [0-7];
+    D = [0-9];
+    L = [a-zA-Z_];
+    H = [a-fA-F0-9];
+    E = [Ee] [+-]? D+;
+    FS = [fFlL];
+    IS = [uUlL]*;
+    ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
+    PP = "#" [ \t]*;
+    NEWLINE = "\r\n" | "\r" | "\n";
+    WHITESPACE = [ \t\v\f]+;
+*/
+
+/*!re2c
+    "/*"            { goto multilinecomment; }
+    "//"            { goto singlelinecomment; }
+
+    L (L|D)*        { RET(TOKEN_IDENTIFIER); }
+    
+    ("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
+    (['] (ESC|any\[\n\\'])* ['])
+                    { RET(TOKEN_INT_LITERAL); }
+    
+    (D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
+                    { RET(TOKEN_FLOAT_LITERAL); }
+    
+    (["] (ESC|any\[\n\\"])* ["])
+                    { RET(TOKEN_STRING_LITERAL); }
+    
+    "..."           { RET(TOKEN_ELLIPSIS); }
+    ">>"            { RET(TOKEN_RSHIFT); }
+    "<<"            { RET(TOKEN_LSHIFT); }
+    "&&"            { RET(TOKEN_ANDAND); }
+    "||"            { RET(TOKEN_OROR); }
+    "<="            { RET(TOKEN_LEQ); }
+    ">="            { RET(TOKEN_GEQ); }
+    "=="            { RET(TOKEN_EQL); }
+    "!="            { RET(TOKEN_NEQ); }
+    "##"            { RET(TOKEN_HASHHASH); }
+    "("             { RET('('); }
+    ")"             { RET(')'); }
+    "["             { RET('['); }
+    "]"             { RET(']'); }
+    "."             { RET('.'); }
+    ","             { RET(','); }
+    "&"             { RET('&'); }
+    "!"             { RET('!'); }
+    "~"             { RET('~'); }
+    "-"             { RET('-'); }
+    "+"             { RET('+'); }
+    "*"             { RET('*'); }
+    "/"             { RET('/'); }
+    "%"             { RET('%'); }
+    "<"             { RET('<'); }
+    ">"             { RET('>'); }
+    "^"             { RET('^'); }
+    "|"             { RET('|'); }
+    ":"             { RET(':'); }
+    ";"             { RET(';'); }
+    "{"             { RET('{'); }
+    "}"             { RET('}'); }
+    "="             { RET('='); }
+    "?"             { RET('?'); }
+    "\\"            { RET('\\'); }
+    "#"             { RET('#'); }
+
+    PP "include"    { RET(TOKEN_PP_INCLUDE); }
+    PP "line"       { RET(TOKEN_PP_LINE); }
+    PP "define"     { RET(TOKEN_PP_DEFINE); }
+    PP "undef"      { RET(TOKEN_PP_UNDEF); }
+    PP "if"         { RET(TOKEN_PP_IF); }
+    PP "ifdef"      { RET(TOKEN_PP_IFDEF); }
+    PP "ifndef"     { RET(TOKEN_PP_IFNDEF); }
+    PP "else"       { RET(TOKEN_PP_ELSE); }
+    PP "elif"       { RET(TOKEN_PP_ELIF); }
+    PP "endif"      { RET(TOKEN_PP_ENDIF); }
+    PP "error"      { RET(TOKEN_PP_ERROR); }
+
+    WHITESPACE      { goto scanner_loop; }
+    NEWLINE         { s->line++; goto scanner_loop; }
+    any             { printf("bad char\n"); goto scanner_loop; }
+*/
+
+multilinecomment:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_PP_INCOMPLETE_COMMENT);
+// The "*\/" is just to avoid screwing up text editor syntax highlighting.
+/*!re2c
+    "*\/"           { goto scanner_loop; }
+    NEWLINE         { s->line++; goto multilinecomment; }
+    any             { goto multilinecomment; }
+*/
+
+singlelinecomment:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_EOI);
+/*!re2c
+    NEWLINE         { s->line++; goto scanner_loop; }
+    any             { goto singlelinecomment; }
+*/
+
+// !!! FIXME
+/*
+bad_chars:
+    if (YYLIMIT == YYCURSOR)
+        RET(TOKEN_BAD_TOKEN);
+*/
+
+/*!re2c
+    NEWLINE         { s->line++; goto scanner_loop; }
+    WHITESPACE      { goto scanner_loop; }
+    any             { goto singlelinecomment; }
+*/
+
+    assert(0 && "Shouldn't hit this code");
+    RET(TOKEN_UNKNOWN);
+} // preprocessor_internal_lexer
+
+// end of mojoshader_lexer_preprocessor.re (or .c) ...
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mojoshader_preprocessor.c	Mon Feb 09 17:53:54 2009 -0500
@@ -0,0 +1,755 @@
+/**
+ * MojoShader; generate shader programs from bytecode of compiled
+ *  Direct3D shaders.
+ *
+ * Please see the file LICENSE.txt in the source's root directory.
+ *
+ *  This file written by Ryan C. Gordon.
+ */
+
+#define __MOJOSHADER_INTERNAL__ 1
+#include "mojoshader_internal.h"
+
+typedef struct DefineHash
+{
+    MOJOSHADER_preprocessorDefine define;
+    struct DefineHash *next;
+} DefineHash;
+
+typedef struct Context
+{
+    int isfail;
+    int out_of_memory;
+    char failstr[128];
+    IncludeState *include_stack;
+    DefineHash *define_hashtable[256];
+    int pushedback;
+    const char *token;
+    unsigned int tokenlen;
+    MOJOSHADER_includeOpen open_callback;
+    MOJOSHADER_includeClose close_callback;
+    MOJOSHADER_malloc malloc;
+    MOJOSHADER_free free;
+    void *malloc_data;
+} Context;
+
+
+// Convenience functions for allocators...
+
+static inline void out_of_memory(Context *ctx)
+{
+    ctx->isfail = ctx->out_of_memory = 1;
+} // out_of_memory
+
+static inline void *Malloc(Context *ctx, const size_t len)
+{
+    void *retval = ctx->malloc((int) len, ctx->malloc_data);
+    if (retval == NULL)
+        out_of_memory(ctx);
+    return retval;
+} // Malloc
+
+static inline void Free(Context *ctx, void *ptr)
+{
+    if (ptr != NULL)  // check for NULL in case of dumb free() impl.
+        ctx->free(ptr, ctx->malloc_data);
+} // Free
+
+static inline char *StrDup(Context *ctx, const char *str)
+{
+    char *retval = (char *) Malloc(ctx, strlen(str) + 1);
+    if (retval != NULL)
+        strcpy(retval, str);
+    return retval;
+} // StrDup
+
+static void failf(Context *ctx, const char *fmt, ...) ISPRINTF(2,3);
+static void failf(Context *ctx, const char *fmt, ...)
+{
+    ctx->isfail = 1;
+    va_list ap;
+    va_start(ap, fmt);
+    vsnprintf(ctx->failstr, sizeof (ctx->failstr), fmt, ap);  // rebuild it.
+    va_end(ap);
+} // failf
+
+static inline void fail(Context *ctx, const char *reason)
+{
+    failf(ctx, "%s", reason);
+} // fail
+
+
+// Preprocessor define hashtable stuff...
+
+static unsigned char hash_define(const char *sym)
+{
+    unsigned char retval = 0;
+    while (sym)
+        retval += *(sym++);
+    return retval;
+} // hash_define
+
+
+static int add_define(Context *ctx, const char *sym, const char *val)
+{
+    char *identifier = NULL;
+    char *definition = NULL;
+    const unsigned char hash = hash_define(sym);
+    DefineHash *bucket = ctx->define_hashtable[hash];
+    while (bucket)
+    {
+        if (strcmp(bucket->define.identifier, sym) == 0)
+        {
+            failf(ctx, "'%s' already defined", sym);
+            return 0;
+        } // if
+        bucket = bucket->next;
+    } // while
+
+    bucket = (DefineHash *) Malloc(ctx, sizeof (DefineHash));
+    if (bucket == NULL)
+        return 0;
+
+    identifier = (char *) Malloc(ctx, strlen(sym) + 1);
+    definition = (char *) Malloc(ctx, strlen(val) + 1);
+    if ((identifier == NULL) || (definition == NULL))
+    {
+        Free(ctx, identifier);
+        Free(ctx, definition);
+        Free(ctx, bucket);
+        return 0;
+    } // if
+
+    strcpy(identifier, sym);
+    strcpy(definition, val);
+    bucket->define.identifier = identifier;
+    bucket->define.definition = definition;
+    bucket->next = ctx->define_hashtable[hash];
+    ctx->define_hashtable[hash] = bucket;
+    return 1;
+} // add_define
+
+
+static int remove_define(Context *ctx, const char *sym)
+{
+    const unsigned char hash = hash_define(sym);
+    DefineHash *bucket = ctx->define_hashtable[hash];
+    DefineHash *prev = NULL;
+    while (bucket)
+    {
+        if (strcmp(bucket->define.identifier, sym) == 0)
+        {
+            if (prev == NULL)
+                ctx->define_hashtable[hash] = bucket->next;
+            else
+                prev->next = bucket->next;
+            Free(ctx, (void *) bucket->define.identifier);
+            Free(ctx, (void *) bucket->define.definition);
+            Free(ctx, bucket);
+            return 1;
+        } // if
+        prev = bucket;
+        bucket = bucket->next;
+    } // while
+
+    failf(ctx, "'%s' not defined", sym);
+    return 0;
+} // remove_define
+
+
+static const char *find_define(Context *ctx, const char *sym)
+{
+    const unsigned char hash = hash_define(sym);
+    DefineHash *bucket = ctx->define_hashtable[hash];
+    while (bucket)
+    {
+        if (strcmp(bucket->define.identifier, sym) == 0)
+            return bucket->define.definition;
+        bucket = bucket->next;
+    } // while
+    return NULL;
+} // find_define
+
+
+static void free_all_defines(Context *ctx)
+{
+    int i;
+    for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)
+    {
+        DefineHash *bucket = ctx->define_hashtable[i];
+        ctx->define_hashtable[i] = NULL;
+        while (bucket)
+        {
+            DefineHash *next = bucket->next;
+            Free(ctx, (void *) bucket->define.identifier);
+            Free(ctx, (void *) bucket->define.definition);
+            Free(ctx, bucket);
+            bucket = next;
+        } // while
+    } // for
+} // find_define
+
+
+static int push_source(Context *ctx, const char *fname, const char *source,
+                       unsigned int srclen, int included)
+{
+    IncludeState *state = (IncludeState *) Malloc(ctx, sizeof (IncludeState));
+    if (state == NULL)
+        return 0;
+    memset(state, '\0', sizeof (IncludeState));
+
+    state->filename = StrDup(ctx, fname);
+    if (state->filename == NULL)
+    {
+        Free(ctx, state);
+        return 0;
+    } // if
+
+    state->included = included;
+    state->source_base = source;
+    state->source = source;
+    state->token = source;
+    state->insert_token = TOKEN_UNKNOWN;
+    state->insert_token2 = TOKEN_UNKNOWN;
+    state->bytes_left = srclen;
+    state->line = 1;
+    state->next = ctx->include_stack;
+
+    ctx->include_stack = state;
+
+    return 1;
+} // push_source
+
+
+static void pop_source(Context *ctx)
+{
+    IncludeState *state = ctx->include_stack;
+    if (state == NULL)
+        return;
+
+    if (state->included)
+    {
+        ctx->close_callback(state->source_base, ctx->malloc,
+                            ctx->free, ctx->malloc_data);
+    } // if
+
+    ctx->include_stack = state->next;
+    Free(ctx, state->filename);
+    Free(ctx, state);
+} // pop_source
+
+
+Preprocessor *preprocessor_start(const char *fname, const char *source,
+                            unsigned int sourcelen,
+                            MOJOSHADER_includeOpen open_callback,
+                            MOJOSHADER_includeClose close_callback,
+                            const MOJOSHADER_preprocessorDefine **defines,
+                            unsigned int define_count,
+                            MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
+{
+    int okay = 1;
+    int i = 0;
+
+    // the preprocessor is internal-only, so we verify all these are != NULL.
+    assert(m != NULL);
+    assert(f != NULL);
+    assert(open_callback != NULL);
+    assert(close_callback != NULL);
+
+    Context *ctx = (Context *) m(sizeof (Context), d);
+    if (ctx == NULL)
+        return 0;
+
+    memset(ctx, '\0', sizeof (Context));
+    ctx->malloc = m;
+    ctx->free = f;
+    ctx->malloc_data = d;
+    ctx->open_callback = open_callback;
+    ctx->close_callback = close_callback;
+
+    for (i = 0; i < define_count; i++)
+    {
+        if (!add_define(ctx, defines[i]->identifier, defines[i]->definition))
+        {
+            okay = 0;
+            break;
+        } // if
+    } // for
+
+    if ((okay) && (!push_source(ctx, fname, source, sourcelen, 0)))
+        okay = 0;
+
+    if (!okay)
+    {
+        preprocessor_end((Preprocessor *) ctx);
+        return NULL;
+    } // if
+
+    return (Preprocessor *) ctx;
+} // preprocessor_start
+
+
+void preprocessor_end(Preprocessor *_ctx)
+{
+    Context *ctx = (Context *) _ctx;
+    if (ctx == NULL)
+        return;
+
+    while (ctx->include_stack != NULL)
+        pop_source(ctx);
+
+    free_all_defines(ctx);
+
+    Free(ctx, ctx);
+} // preprocessor_end
+
+
+void preprocessor_clearerror(Preprocessor *_ctx)
+{
+    Context *ctx = (Context *) _ctx;
+    ctx->isfail = 0;
+} // preprocessor_clearerror
+
+
+const char *preprocessor_error(Preprocessor *_ctx)
+{
+    Context *ctx = (Context *) _ctx;
+    return ctx->isfail ? ctx->failstr : NULL;
+} // preprocessor_error
+
+
+int preprocessor_outofmemory(Preprocessor *_ctx)
+{
+    Context *ctx = (Context *) _ctx;
+    return ctx->out_of_memory;
+} // preprocessor_outofmemory
+
+
+const char *preprocessor_nexttoken(Preprocessor *_ctx, unsigned int *_len,
+                                    Token *_token)
+{
+    Context *ctx = (Context *) _ctx;
+
+    while (1)
+    {
+        IncludeState *state = ctx->include_stack;
+        if (state == NULL)
+            return NULL;  // we're done!
+
+        if (state->insert_token != TOKEN_UNKNOWN)
+        {
+            state->insert_tokchar = (char) state->insert_token;
+            *_token = state->insert_token;
+            *_len = 1;
+            state->insert_token = TOKEN_UNKNOWN;
+            return &state->insert_tokchar;
+        } // if
+
+        else if (state->insert_token2 != TOKEN_UNKNOWN)
+        {
+            state->insert_tokchar = (char) state->insert_token2;
+            *_token = state->insert_token2;
+            *_len = 1;
+            state->insert_token2 = TOKEN_UNKNOWN;
+            return &state->insert_tokchar;
+        } // if
+
+        Token token = preprocessor_internal_lexer(state);
+        if (token == TOKEN_EOI)
+        {
+            assert(state->bytes_left == 0);
+            pop_source(ctx);
+            continue;  // pick up again after parent's #include line.
+        } // if
+
+        // Microsoft's preprocessor is weird.
+        // It ignores newlines, and then inserts its own around certain
+        //  tokens. For example, after a semicolon. This allows HLSL code to
+        //  be mostly readable, and lets the ';' work as a single line comment
+        //  in the assembler.
+        if ( (token == ((Token) ';')) || (token == ((Token) '}')) )
+            state->insert_token = (Token) '\n';
+        else if (token == ((Token) '{'))
+        {
+            state->insert_token = (Token) '{';
+            state->insert_token2 = (Token) '\n';
+            state->insert_tokchar = '\n';
+            *_token = (Token) '\n';
+            *_len = 1;
+            return &state->insert_tokchar;
+        } // else if
+
+        *_token = token;
+        *_len = (unsigned int) (state->source - state->token);
+        return state->token;
+    } // while
+} // preprocessor_nexttoken
+
+
+const char *preprocessor_sourcepos(Preprocessor *_ctx, unsigned int *pos)
+{
+    Context *ctx = (Context *) _ctx;
+    if (ctx->include_stack == NULL)
+    {
+        *pos = 0;
+        return NULL;
+    } // if
+
+    *pos = ctx->include_stack->line;
+    return ctx->include_stack->filename;
+} // preprocessor_sourcepos
+
+
+// public API...
+
+static const MOJOSHADER_preprocessData out_of_mem_data_preprocessor = {
+    1, &MOJOSHADER_out_of_mem_error, 0, 0, 0, 0, 0
+};
+
+#define BUFFER_LEN (64 * 1024)
+typedef struct BufferList
+{
+    char buffer[BUFFER_LEN];
+    size_t bytes;
+    struct BufferList *next;
+} BufferList;
+
+typedef struct Buffer
+{
+    size_t total_bytes;
+    BufferList head;
+    BufferList *tail;
+} Buffer;
+
+static void buffer_init(Buffer *buffer)
+{
+    buffer->total_bytes = 0;
+    buffer->head.bytes = 0;
+    buffer->head.next = NULL;
+    buffer->tail = &buffer->head;
+} // buffer_init
+
+
+static int add_to_buffer(Buffer *buffer, const char *data,
+                         size_t len, MOJOSHADER_malloc m, void *d)
+{
+    buffer->total_bytes += len;
+    while (len > 0)
+    {
+        const size_t avail = BUFFER_LEN - buffer->tail->bytes;
+        const size_t cpy = (avail > len) ? len : avail;
+        memcpy(buffer->tail->buffer + buffer->tail->bytes, data, cpy);
+        len -= cpy;
+        data += cpy;
+        buffer->tail->bytes += cpy;
+        assert(buffer->tail->bytes <= BUFFER_LEN);
+        if (buffer->tail->bytes == BUFFER_LEN)
+        {
+            BufferList *item = (BufferList *) m(sizeof (BufferList), d);
+            if (item == NULL)
+                return 0;
+            item->bytes = 0;
+            item->next = NULL;
+            buffer->tail->next = item;
+            buffer->tail = item;
+        } // if
+    } // while
+
+    return 1;
+} // add_to_buffer
+
+
+static int indent_buffer(Buffer *buffer, int n, int newline,
+                         MOJOSHADER_malloc m, void *d)
+{
+    static char spaces[4] = { ' ', ' ', ' ', ' ' };
+    if (newline)
+    {
+        while (n--)
+        {
+            if (!add_to_buffer(buffer, spaces, sizeof (spaces), m, d))
+                return 0;
+        } // while
+    } // if
+    else
+    {
+        if (!add_to_buffer(buffer, spaces, 1, m, d))
+            return 0;
+    } // else
+    return 1;
+} // indent_buffer
+
+
+static char *flatten_buffer(Buffer *buffer, MOJOSHADER_malloc m, void *d)
+{
+    char *retval = m(buffer->total_bytes + 1, d);
+    if (retval == NULL)
+        return NULL;
+    BufferList *item = &buffer->head;
+    char *ptr = retval;
+    while (item != NULL)
+    {
+        BufferList *next = item->next;
+        memcpy(ptr, item->buffer, item->bytes);
+        ptr += item->bytes;
+        item = next;
+    } // while
+    *ptr = '\0';
+
+    assert(ptr == (retval + buffer->total_bytes));
+    return retval;
+} // flatten_buffer
+
+
+static void free_buffer(Buffer *buffer, MOJOSHADER_free f, void *d)
+{
+    // head is statically allocated, so start with head.next...
+    BufferList *item = buffer->head.next;
+    while (item != NULL)
+    {
+        BufferList *next = item->next;
+        f(item, d);
+        item = next;
+    } // while
+    buffer_init(buffer);
+} // free_buffer
+
+
+// !!! FIXME: cut and paste.
+static void free_error_list(ErrorList *item, MOJOSHADER_free f, void *d)
+{
+    while (item != NULL)
+    {
+        ErrorList *next = item->next;
+        f((void *) item->error.error, d);
+        f((void *) item->error.filename, d);
+        f(item, d);
+        item = next;
+    } // while
+} // free_error_list
+
+
+// !!! FIXME: cut and paste.
+static MOJOSHADER_error *build_errors(ErrorList **errors, const int count,
+                         MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
+{
+    int total = 0;
+    MOJOSHADER_error *retval = (MOJOSHADER_error *)
+                                m(sizeof (MOJOSHADER_error) * count, d);
+    if (retval == NULL)
+        return NULL;
+
+    ErrorList *item = *errors;
+    while (item != NULL)
+    {
+        ErrorList *next = item->next;
+        // reuse the string allocations
+        memcpy(&retval[total], &item->error, sizeof (MOJOSHADER_error));
+        f(item, d);
+        item = next;
+        total++;
+    } // while
+    *errors = NULL;
+
+    assert(total == count);
+    return retval;
+} // build_errors
+
+
+const MOJOSHADER_preprocessData *MOJOSHADER_preprocess(const char *source,
+                             unsigned int sourcelen,
+                             const MOJOSHADER_preprocessorDefine **defines,
+                             unsigned int define_count,
+                             MOJOSHADER_includeOpen include_open,
+                             MOJOSHADER_includeClose include_close,
+                             MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
+{
+    ErrorList *errors = NULL;
+    int error_count = 0;
+
+    if (m == NULL) m = MOJOSHADER_internal_malloc;
+    if (f == NULL) f = MOJOSHADER_internal_free;
+
+include_open = (MOJOSHADER_includeOpen) 0x1;
+include_close = (MOJOSHADER_includeClose) 0x1;
+
+    const char *fname = "*";  // !!! FIXME
+    Preprocessor *pp = preprocessor_start(fname, source, sourcelen,
+                                          include_open, include_close,
+                                          defines, define_count, m, f, d);
+
+    if (pp == NULL)
+        return &out_of_mem_data_preprocessor;
+
+    Token token = TOKEN_UNKNOWN;
+    const char *tokstr = NULL;
+    const char *err = NULL;
+
+    Buffer buffer;
+    buffer_init(&buffer);
+
+    int nl = 1;
+    int indent = 0;
+    unsigned int len = 0;
+    while ((tokstr = preprocessor_nexttoken(pp, &len, &token)) != NULL)
+    {
+        #ifdef _WINDOWS
+        static const char endline[] = { '\r', '\n' };
+        #else
+        static const char endline[] = { '\n' };
+        #endif
+
+        const int isnewline = (token == ((Token) '\n'));
+        if (isnewline)
+        {
+            tokstr = endline;  // convert to platform-specific.
+            len = sizeof (endline);
+        } // if
+
+        if ((token == ((Token) '}')) && (indent > 0))
+            indent--;
+
+        int out_of_memory = preprocessor_outofmemory(pp);
+
+        if ((!out_of_memory) && (!isnewline))
+            out_of_memory = !indent_buffer(&buffer, indent, nl, m, d);
+
+        if (!out_of_memory)
+            out_of_memory = !add_to_buffer(&buffer, tokstr, len, m, d);
+
+        if (token == ((Token) '{'))
+            indent++;
+
+        nl = isnewline;
+
+        if ((!out_of_memory) && ((err = preprocessor_error(pp)) != NULL))
+        {
+            ErrorList *error = (ErrorList *) m(sizeof (ErrorList), d);
+            unsigned int pos = 0;
+            char *fname = NULL;
+            const char *str = preprocessor_sourcepos(pp, &pos);
+            if (str != NULL)
+            {
+                fname = (char *) m(strlen(str) + 1, d);
+                if (fname != NULL)
+                    strcpy(fname, str);
+            } // if
+
+            // !!! FIXME: cut and paste with other error handlers.
+            char *errstr = (char *) m(strlen(err) + 1, d);
+            if (errstr != NULL)
+                strcpy(errstr, err);
+
+            out_of_memory = ((!error) || ((!fname) && (str)) || (!errstr));
+            if (out_of_memory)
+            {
+                if (errstr) f(errstr, d);
+                if (fname) f(fname, d);
+                if (error) f(error, d);
+            } // if
+            else
+            {
+                error->error.error = errstr;
+                error->error.filename = fname;
+                error->error.error_position = pos;
+                error->next = NULL;
+
+                ErrorList *prev = NULL;
+                ErrorList *item = errors;
+                while (item != NULL)
+                {
+                    prev = item;
+                    item = error->next;
+                } // while
+
+                if (prev == NULL)
+                    errors = error;
+                else
+                    prev->next = error;
+
+                error_count++;
+            } // else
+
+            preprocessor_clearerror(pp);
+            continue;
+        } // if
+
+        if (out_of_memory)
+        {
+            preprocessor_end(pp);
+            free_buffer(&buffer, f, d);
+            free_error_list(errors, f, d);
+            return &out_of_mem_data_preprocessor;
+        } // if
+    } // while
+    
+    preprocessor_end(pp);
+
+    const size_t total_bytes = buffer.total_bytes;
+    char *output = flatten_buffer(&buffer, m, d);
+    free_buffer(&buffer, f, d);
+    if (output == NULL)
+    {
+        free_error_list(errors, f, d);
+        return &out_of_mem_data_preprocessor;
+    } // if
+
+    MOJOSHADER_preprocessData *retval = (MOJOSHADER_preprocessData *)
+                                    m(sizeof (MOJOSHADER_preprocessData), d);
+    if (retval == NULL)
+    {
+        free_error_list(errors, f, d);
+        f(output, d);
+        return &out_of_mem_data_preprocessor;
+    } // if
+
+    retval->errors = build_errors(&errors, error_count, m, f, d);
+    if (retval->errors == NULL)
+    {
+        free_error_list(errors, f, d);
+        f(retval, d);
+        f(output, d);
+        return &out_of_mem_data_preprocessor;
+    } // if
+
+    retval->error_count = error_count;
+    retval->output = output;
+    retval->output_len = total_bytes;
+    retval->malloc = m;
+    retval->free = f;
+    retval->malloc_data = d;
+    return retval;
+} // MOJOSHADER_preprocess
+
+
+void MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData *_data)
+{
+    MOJOSHADER_preprocessData *data = (MOJOSHADER_preprocessData *) _data;
+    if ((data == NULL) || (data == &out_of_mem_data_preprocessor))
+        return;
+
+    MOJOSHADER_free f = (data->free == NULL) ? MOJOSHADER_internal_free : data->free;
+    void *d = data->malloc_data;
+    int i;
+
+    if (data->output != NULL)
+        f((void *) data->output, d);
+
+    if (data->errors != NULL)
+    {
+        for (i = 0; i < data->error_count; i++)
+        {
+            if (data->errors[i].error != NULL)
+                f((void *) data->errors[i].error, d);
+            if (data->errors[i].filename != NULL)
+                f((void *) data->errors[i].filename, d);
+        } // for
+        f(data->errors, d);
+    } // if
+
+    f(data, d);
+} // MOJOSHADER_freePreprocessData
+
+
+// end of mojoshader_preprocessor.c ...
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/preprocess.c	Mon Feb 09 17:53:54 2009 -0500
@@ -0,0 +1,92 @@
+/**
+ * MojoShader; generate shader programs from bytecode of compiled
+ *  Direct3D shaders.
+ *
+ * Please see the file LICENSE.txt in the source's root directory.
+ *
+ *  This file written by Ryan C. Gordon.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "mojoshader.h"
+
+static int preprocess(const char *buf, int len, const char *outfile)
+{
+    FILE *io = fopen(outfile, "wb");
+    if (io == NULL)
+    {
+        printf(" ... fopen('%s') failed.\n", outfile);
+        return 0;
+    } // if
+
+    const MOJOSHADER_preprocessData *pd;
+    int retval = 0;
+
+    pd = MOJOSHADER_preprocess(buf, len, NULL, 0, NULL,
+                               NULL, NULL, NULL, NULL);
+
+    if (pd->error_count > 0)
+    {
+        int i;
+        for (i = 0; i < pd->error_count; i++)
+        {
+            printf("ERROR: (line %d) %s\n",
+                    pd->errors[i].error_position,
+                    pd->errors[i].error);
+        } // for
+    } // if
+    else
+    {
+        if (pd->output != NULL)
+        {
+            if (fwrite(pd->output, pd->output_len, 1, io) != 1)
+                printf(" ... fwrite('%s') failed.\n", outfile);
+            else if (fclose(io) == EOF)
+                printf(" ... fclose('%s') failed.\n", outfile);
+            else
+                retval = 1;
+        } // if
+    } // else
+    MOJOSHADER_freePreprocessData(pd);
+
+    return retval;
+} // assemble
+
+
+int main(int argc, char **argv)
+{
+    int retval = 1;
+
+    if (argc != 3)
+        printf("\n\nUSAGE: %s <inputfile> <outputfile>\n\n", argv[0]);
+    else
+    {
+        const char *infile = argv[1];
+        const char *outfile = argv[2];
+        FILE *io = fopen(infile, "rb");
+        if (io == NULL)
+            printf(" ... fopen('%s') failed.\n", infile);
+        else
+        {
+            char *buf = (char *) malloc(1000000);
+            int rc = fread(buf, 1, 1000000, io);
+            fclose(io);
+            if (rc == EOF)
+                printf(" ... fread('%s') failed.\n", infile);
+            else
+            {
+                if (preprocess(buf, rc, outfile))
+                    retval = 0;
+                else
+                    remove(outfile);
+                free(buf);
+            } // else
+        } // for
+    } // else
+
+    return retval;
+} // main
+
+// end of assemble.c ...
+