mojoshader_lexer.re
changeset 642 624ab8696f9b
parent 635 be3b428802a1
child 648 243e35876142
equal deleted inserted replaced
641:2002d0399ef2 642:624ab8696f9b
    22 #define __MOJOSHADER_INTERNAL__ 1
    22 #define __MOJOSHADER_INTERNAL__ 1
    23 #include "mojoshader_internal.h"
    23 #include "mojoshader_internal.h"
    24 
    24 
    25 typedef unsigned char uchar;
    25 typedef unsigned char uchar;
    26 
    26 
    27 #define RET(t) do { update_state(s, cursor, token); return t; } while (0)
    27 /*!max:re2c */
       
    28 #define RET(t) do { update_state(s, eoi, cursor, token); return t; } while (0)
    28 #define YYCTYPE uchar
    29 #define YYCTYPE uchar
    29 #define YYCURSOR cursor
    30 #define YYCURSOR cursor
    30 #define YYLIMIT limit
    31 #define YYLIMIT limit
    31 #define YYMARKER s->lexer_marker
    32 #define YYMARKER s->lexer_marker
    32 #define YYFILL(n) { if ((n) == 1) { RET(TOKEN_EOI); } }
    33 #define YYFILL(n) { if ((n) == 1) { cursor = sentinel; limit = cursor + YYMAXFILL; eoi = 1; } }
    33 
    34 
    34 static void update_state(IncludeState *s, const uchar *cur, const uchar *tok)
    35 static uchar sentinel[YYMAXFILL];
       
    36 
       
    37 static void update_state(IncludeState *s, int eoi,
       
    38                          const uchar *cur, const uchar *tok)
    35 {
    39 {
    36     s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
    40     if (eoi)
    37     s->source = (const char *) cur;
    41     {
    38     s->token = (const char *) tok;
    42         s->bytes_left = 0;
       
    43         s->source = (const char *) s->source_base + s->orig_length;
       
    44         if ( (tok >= sentinel) && (tok < (sentinel+YYMAXFILL)) )
       
    45             s->token = s->source;
       
    46         else
       
    47             s->token = (const char *) tok;
       
    48     } // if
       
    49     else
       
    50     {
       
    51         s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
       
    52         s->source = (const char *) cur;
       
    53         s->token = (const char *) tok;
       
    54     } // else
    39 } // update_state
    55 } // update_state
    40 
    56 
    41 Token preprocessor_internal_lexer(IncludeState *s)
    57 Token preprocessor_internal_lexer(IncludeState *s)
    42 {
    58 {
    43     const uchar *cursor = (const uchar *) s->source;
    59     const uchar *cursor = (const uchar *) s->source;
    44     const uchar *token;
    60     const uchar *token;
    45     const uchar *matchptr;
    61     const uchar *matchptr;
    46     const uchar *limit = cursor + s->bytes_left;
    62     const uchar *limit = cursor + s->bytes_left;
       
    63     int eoi = 0;
    47     int saw_newline = 0;
    64     int saw_newline = 0;
    48 
    65 
    49 scanner_loop:
    66 scanner_loop:
       
    67     if (YYLIMIT == YYCURSOR) YYFILL(1);
    50     token = cursor;
    68     token = cursor;
    51 
       
    52     if (YYLIMIT == YYCURSOR)
       
    53         RET(TOKEN_EOI);
       
    54 
    69 
    55 /*!re2c
    70 /*!re2c
    56     ANY = [\000-\377];
    71     ANY = [\000-\377];
    57     ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
    72     ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
    58     O = [0-7];
    73     O = [0-7];
   130     "{"             { RET('{'); }
   145     "{"             { RET('{'); }
   131     "}"             { RET('}'); }
   146     "}"             { RET('}'); }
   132     "="             { RET('='); }
   147     "="             { RET('='); }
   133     "?"             { RET('?'); }
   148     "?"             { RET('?'); }
   134 
   149 
       
   150     "\000"          { if (eoi) { RET(TOKEN_EOI); } goto bad_chars; }
       
   151 
   135     PP "include"    { RET(TOKEN_PP_INCLUDE); }
   152     PP "include"    { RET(TOKEN_PP_INCLUDE); }
   136     PP "line"       { RET(TOKEN_PP_LINE); }
   153     PP "line"       { RET(TOKEN_PP_LINE); }
   137     PP "define"     { RET(TOKEN_PP_DEFINE); }
   154     PP "define"     { RET(TOKEN_PP_DEFINE); }
   138     PP "undef"      { RET(TOKEN_PP_UNDEF); }
   155     PP "undef"      { RET(TOKEN_PP_UNDEF); }
   139     PP "if"         { RET(TOKEN_PP_IF); }
   156     PP "if"         { RET(TOKEN_PP_IF); }
   148     NEWLINE         { s->line++; RET('\n'); }
   165     NEWLINE         { s->line++; RET('\n'); }
   149     ANY             { goto bad_chars; }
   166     ANY             { goto bad_chars; }
   150 */
   167 */
   151 
   168 
   152 multilinecomment:
   169 multilinecomment:
   153     if (YYLIMIT == YYCURSOR)
   170     if (YYLIMIT == YYCURSOR) YYFILL(1);
   154         RET(TOKEN_INCOMPLETE_COMMENT);
       
   155     matchptr = cursor;
   171     matchptr = cursor;
   156 // The "*\/" is just to avoid screwing up text editor syntax highlighting.
   172 // The "*\/" is just to avoid screwing up text editor syntax highlighting.
   157 /*!re2c
   173 /*!re2c
   158     "*\/"           {
   174     "*\/"           {
   159                         if (saw_newline)
   175                         if (saw_newline)
   166                         s->line++;
   182                         s->line++;
   167                         token = matchptr;
   183                         token = matchptr;
   168                         saw_newline = 1;
   184                         saw_newline = 1;
   169                         goto multilinecomment;
   185                         goto multilinecomment;
   170                     }
   186                     }
       
   187     "\000"          {
       
   188                         if (eoi)
       
   189                             RET(TOKEN_INCOMPLETE_COMMENT);
       
   190                         goto multilinecomment;
       
   191                     }
   171     ANY             { goto multilinecomment; }
   192     ANY             { goto multilinecomment; }
   172 */
   193 */
   173 
   194 
   174 singlelinecomment:
   195 singlelinecomment:
   175     if (YYLIMIT == YYCURSOR)
   196     if (YYLIMIT == YYCURSOR) YYFILL(1);
   176         RET(TOKEN_EOI);
       
   177     matchptr = cursor;
   197     matchptr = cursor;
   178 /*!re2c
   198 /*!re2c
   179     NEWLINE         { s->line++; token = matchptr; RET('\n'); }
   199     NEWLINE         { s->line++; token = matchptr; RET('\n'); }
       
   200     "\000"          { if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
   180     ANY             { goto singlelinecomment; }
   201     ANY             { goto singlelinecomment; }
   181 */
   202 */
   182 
   203 
   183 bad_chars:
   204 bad_chars:
   184     if (YYLIMIT == YYCURSOR)
   205     if (YYLIMIT == YYCURSOR) YYFILL(1);
   185         RET(TOKEN_BAD_CHARS);
       
   186 
       
   187 /*!re2c
   206 /*!re2c
   188     ANYLEGAL        { cursor--; RET(TOKEN_BAD_CHARS); }
   207     ANYLEGAL        { cursor--; RET(TOKEN_BAD_CHARS); }
       
   208     "\000"          {
       
   209                         if (eoi)
       
   210                         {
       
   211                             assert( !((token >= sentinel) &&
       
   212                                      (token < sentinel+YYMAXFILL)) );
       
   213                             eoi = 0;
       
   214                             cursor = (uchar *) s->source_base + s->orig_length;
       
   215                             RET(TOKEN_BAD_CHARS);  // next call will be EOI.
       
   216                         }
       
   217                         goto bad_chars;
       
   218                     }
       
   219 
   189     ANY             { goto bad_chars; }
   220     ANY             { goto bad_chars; }
   190 */
   221 */
   191 
   222 
   192     assert(0 && "Shouldn't hit this code");
   223     assert(0 && "Shouldn't hit this code");
   193     RET(TOKEN_UNKNOWN);
   224     RET(TOKEN_UNKNOWN);