mojoshader_lexer.re
changeset 555 940821555fda
child 564 c669568326fb
equal deleted inserted replaced
554:42dd28107cd8 555:940821555fda
       
     1 /**
       
     2  * MojoShader; generate shader programs from bytecode of compiled
       
     3  *  Direct3D shaders.
       
     4  *
       
     5  * Please see the file LICENSE.txt in the source's root directory.
       
     6  *
       
     7  *  This file written by Ryan C. Gordon.
       
     8  */
       
     9 
       
    10 // This was originally based on examples/pp-c.re from re2c: http://re2c.org/
       
    11 //   re2c is public domain code.
       
    12 //
       
    13 // You build mojoshader_lexer_preprocessor.c from the .re file with re2c...
       
    14 // re2c -is -o mojoshader_lexer_preprocessor.c mojoshader_lexer_preprocessor.re
       
    15 //
       
    16 // Changes to the lexer are done to the .re file, not the C code!
       
    17 //
       
    18 // Please note that this isn't a perfect C lexer, since it is used for both
       
    19 //  HLSL and shader assembly language, and follows the quirks of Microsoft's
       
    20 //  tools.
       
    21 
       
    22 #define __MOJOSHADER_INTERNAL__ 1
       
    23 #include "mojoshader_internal.h"
       
    24 
       
    25 typedef unsigned char uchar;
       
    26 
       
    27 #define RET(t) { update_state(s, cursor, token); return t; }
       
    28 #define YYCTYPE uchar
       
    29 #define YYCURSOR cursor
       
    30 #define YYLIMIT limit
       
    31 #define YYMARKER s->lexer_marker
       
    32 #define YYFILL(n) { if ((n) == 1) { RET(TOKEN_EOI); } }
       
    33 
       
    34 static void update_state(IncludeState *s, const uchar *cur, const uchar *tok)
       
    35 {
       
    36     s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
       
    37     s->source = (const char *) cur;
       
    38     s->token = (const char *) tok;
       
    39 } // update_state
       
    40 
       
    41 Token preprocessor_internal_lexer(IncludeState *s)
       
    42 {
       
    43     const uchar *cursor = (const uchar *) s->source;
       
    44     const uchar *token;
       
    45     const uchar *limit = cursor + s->bytes_left;
       
    46 
       
    47 scanner_loop:
       
    48     token = cursor;
       
    49 
       
    50     if (YYLIMIT == YYCURSOR)
       
    51         RET(TOKEN_EOI);
       
    52 
       
    53 /*!re2c
       
    54     any = [\000-\377];
       
    55     O = [0-7];
       
    56     D = [0-9];
       
    57     L = [a-zA-Z_];
       
    58     H = [a-fA-F0-9];
       
    59     E = [Ee] [+-]? D+;
       
    60     FS = [fFlL];
       
    61     IS = [uUlL]*;
       
    62     ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
       
    63     PP = "#" [ \t]*;
       
    64     NEWLINE = "\r\n" | "\r" | "\n";
       
    65     WHITESPACE = [ \t\v\f]+;
       
    66 */
       
    67 
       
    68 /*!re2c
       
    69     "/*"            { goto multilinecomment; }
       
    70     "//"            { goto singlelinecomment; }
       
    71 
       
    72     L (L|D)*        { RET(TOKEN_IDENTIFIER); }
       
    73     
       
    74     ("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
       
    75     (['] (ESC|any\[\n\\'])* ['])
       
    76                     { RET(TOKEN_INT_LITERAL); }
       
    77     
       
    78     (D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
       
    79                     { RET(TOKEN_FLOAT_LITERAL); }
       
    80     
       
    81     (["] (ESC|any\[\n\\"])* ["])
       
    82                     { RET(TOKEN_STRING_LITERAL); }
       
    83     
       
    84     "..."           { RET(TOKEN_ELLIPSIS); }
       
    85     ">>"            { RET(TOKEN_RSHIFT); }
       
    86     "<<"            { RET(TOKEN_LSHIFT); }
       
    87     "&&"            { RET(TOKEN_ANDAND); }
       
    88     "||"            { RET(TOKEN_OROR); }
       
    89     "<="            { RET(TOKEN_LEQ); }
       
    90     ">="            { RET(TOKEN_GEQ); }
       
    91     "=="            { RET(TOKEN_EQL); }
       
    92     "!="            { RET(TOKEN_NEQ); }
       
    93     "##"            { RET(TOKEN_HASHHASH); }
       
    94     "("             { RET('('); }
       
    95     ")"             { RET(')'); }
       
    96     "["             { RET('['); }
       
    97     "]"             { RET(']'); }
       
    98     "."             { RET('.'); }
       
    99     ","             { RET(','); }
       
   100     "&"             { RET('&'); }
       
   101     "!"             { RET('!'); }
       
   102     "~"             { RET('~'); }
       
   103     "-"             { RET('-'); }
       
   104     "+"             { RET('+'); }
       
   105     "*"             { RET('*'); }
       
   106     "/"             { RET('/'); }
       
   107     "%"             { RET('%'); }
       
   108     "<"             { RET('<'); }
       
   109     ">"             { RET('>'); }
       
   110     "^"             { RET('^'); }
       
   111     "|"             { RET('|'); }
       
   112     ":"             { RET(':'); }
       
   113     ";"             { RET(';'); }
       
   114     "{"             { RET('{'); }
       
   115     "}"             { RET('}'); }
       
   116     "="             { RET('='); }
       
   117     "?"             { RET('?'); }
       
   118     "\\"            { RET('\\'); }
       
   119     "#"             { RET('#'); }
       
   120 
       
   121     PP "include"    { RET(TOKEN_PP_INCLUDE); }
       
   122     PP "line"       { RET(TOKEN_PP_LINE); }
       
   123     PP "define"     { RET(TOKEN_PP_DEFINE); }
       
   124     PP "undef"      { RET(TOKEN_PP_UNDEF); }
       
   125     PP "if"         { RET(TOKEN_PP_IF); }
       
   126     PP "ifdef"      { RET(TOKEN_PP_IFDEF); }
       
   127     PP "ifndef"     { RET(TOKEN_PP_IFNDEF); }
       
   128     PP "else"       { RET(TOKEN_PP_ELSE); }
       
   129     PP "elif"       { RET(TOKEN_PP_ELIF); }
       
   130     PP "endif"      { RET(TOKEN_PP_ENDIF); }
       
   131     PP "error"      { RET(TOKEN_PP_ERROR); }
       
   132 
       
   133     WHITESPACE      { goto scanner_loop; }
       
   134     NEWLINE         { s->line++; goto scanner_loop; }
       
   135     any             { printf("bad char\n"); goto scanner_loop; }
       
   136 */
       
   137 
       
   138 multilinecomment:
       
   139     if (YYLIMIT == YYCURSOR)
       
   140         RET(TOKEN_PP_INCOMPLETE_COMMENT);
       
   141 // The "*\/" is just to avoid screwing up text editor syntax highlighting.
       
   142 /*!re2c
       
   143     "*\/"           { goto scanner_loop; }
       
   144     NEWLINE         { s->line++; goto multilinecomment; }
       
   145     any             { goto multilinecomment; }
       
   146 */
       
   147 
       
   148 singlelinecomment:
       
   149     if (YYLIMIT == YYCURSOR)
       
   150         RET(TOKEN_EOI);
       
   151 /*!re2c
       
   152     NEWLINE         { s->line++; goto scanner_loop; }
       
   153     any             { goto singlelinecomment; }
       
   154 */
       
   155 
       
   156 // !!! FIXME
       
   157 /*
       
   158 bad_chars:
       
   159     if (YYLIMIT == YYCURSOR)
       
   160         RET(TOKEN_BAD_TOKEN);
       
   161 */
       
   162 
       
   163 /*!re2c
       
   164     NEWLINE         { s->line++; goto scanner_loop; }
       
   165     WHITESPACE      { goto scanner_loop; }
       
   166     any             { goto singlelinecomment; }
       
   167 */
       
   168 
       
   169     assert(0 && "Shouldn't hit this code");
       
   170     RET(TOKEN_UNKNOWN);
       
   171 } // preprocessor_internal_lexer
       
   172 
       
   173 // end of mojoshader_lexer_preprocessor.re (or .c) ...
       
   174