mojoshader_lexer.re
author Ethan Lee <flibitijibibo@flibitijibibo.com>
Tue, 25 Aug 2020 22:41:43 -0400
changeset 1300 f1cdc5187d53
parent 1147 ff14a55da268
permissions -rw-r--r--
Handle MOJOSHADER_USAGE_UNKNOWN in SPIR-V linker
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     1
/**
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     2
 * MojoShader; generate shader programs from bytecode of compiled
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     3
 *  Direct3D shaders.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     4
 *
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     5
 * Please see the file LICENSE.txt in the source's root directory.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     6
 *
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     7
 *  This file written by Ryan C. Gordon.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     8
 */
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
     9
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    10
// This was originally based on examples/pp-c.re from re2c: http://re2c.org/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    11
//   re2c is public domain code.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    12
//
1147
ff14a55da268 Fixed a few outdated comments.
Ryan C. Gordon <icculus@icculus.org>
parents: 1142
diff changeset
    13
// You build mojoshader_lexer.c from the .re file with re2c...
ff14a55da268 Fixed a few outdated comments.
Ryan C. Gordon <icculus@icculus.org>
parents: 1142
diff changeset
    14
// re2c -is -o mojoshader_lexer.c mojoshader_lexer.re
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    15
//
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    16
// Changes to the lexer are done to the .re file, not the C code!
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    17
//
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    18
// Please note that this isn't a perfect C lexer, since it is used for both
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    19
//  HLSL and shader assembly language, and follows the quirks of Microsoft's
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    20
//  tools.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    21
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    22
#define __MOJOSHADER_INTERNAL__ 1
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    23
#include "mojoshader_internal.h"
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    24
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    25
typedef unsigned char uchar;
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    26
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    27
/*!max:re2c */
798
5dd67cc04cf9 Bunch of small tweaks to make this compile as C++ code without errors/warnings.
Ryan C. Gordon <icculus@icculus.org>
parents: 697
diff changeset
    28
#define RET(t) return update_state(s, eoi, cursor, token, (Token) t)
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    29
#define YYCTYPE uchar
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    30
#define YYCURSOR cursor
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    31
#define YYLIMIT limit
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    32
#define YYMARKER s->lexer_marker
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    33
#define YYFILL(n) { if ((n) == 1) { cursor = sentinel; limit = cursor + YYMAXFILL; eoi = 1; } }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    34
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    35
static uchar sentinel[YYMAXFILL];
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    36
651
05ad72d120a5 Lexer now stores token value to IncludeState.
Ryan C. Gordon <icculus@icculus.org>
parents: 649
diff changeset
    37
static Token update_state(IncludeState *s, int eoi, const uchar *cur,
05ad72d120a5 Lexer now stores token value to IncludeState.
Ryan C. Gordon <icculus@icculus.org>
parents: 649
diff changeset
    38
                          const uchar *tok, const Token val)
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    39
{
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    40
    if (eoi)
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    41
    {
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    42
        s->bytes_left = 0;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    43
        s->source = (const char *) s->source_base + s->orig_length;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    44
        if ( (tok >= sentinel) && (tok < (sentinel+YYMAXFILL)) )
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    45
            s->token = s->source;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    46
        else
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    47
            s->token = (const char *) tok;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    48
    } // if
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    49
    else
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    50
    {
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    51
        s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    52
        s->source = (const char *) cur;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    53
        s->token = (const char *) tok;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    54
    } // else
648
243e35876142 Have lexer calculate token size now.
Ryan C. Gordon <icculus@icculus.org>
parents: 642
diff changeset
    55
    s->tokenlen = (unsigned int) (s->source - s->token);
651
05ad72d120a5 Lexer now stores token value to IncludeState.
Ryan C. Gordon <icculus@icculus.org>
parents: 649
diff changeset
    56
    s->tokenval = val;
05ad72d120a5 Lexer now stores token value to IncludeState.
Ryan C. Gordon <icculus@icculus.org>
parents: 649
diff changeset
    57
    return val;
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    58
} // update_state
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    59
649
527cfe6f2d45 Renamed preprocessor_internal_lexer to preprocessor_lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 648
diff changeset
    60
Token preprocessor_lexer(IncludeState *s)
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    61
{
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    62
    const uchar *cursor = (const uchar *) s->source;
679
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    63
    const uchar *token = cursor;
595
a72705c067dc Handle lexing of comment ends better.
Ryan C. Gordon <icculus@icculus.org>
parents: 594
diff changeset
    64
    const uchar *matchptr;
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    65
    const uchar *limit = cursor + s->bytes_left;
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
    66
    int eoi = 0;
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    67
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    68
/*!re2c
596
8efa254e8840 Renamed "any" to "ANY" to match other rules.
Ryan C. Gordon <icculus@icculus.org>
parents: 595
diff changeset
    69
    ANY = [\000-\377];
597
832dbfa63509 Bogus chars in the lexer now return a token that signifies this.
Ryan C. Gordon <icculus@icculus.org>
parents: 596
diff changeset
    70
    ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    71
    O = [0-7];
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    72
    D = [0-9];
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    73
    L = [a-zA-Z_];
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    74
    H = [a-fA-F0-9];
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    75
    E = [Ee] [+-]? D+;
959
8a5d46db0cc6 HLSL uses h or H as a float literal suffix, meaning "half".
Ryan C. Gordon <icculus@icculus.org>
parents: 910
diff changeset
    76
    FS = [fFhH];
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    77
    IS = [uUlL]*;
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    78
    ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    79
    PP = "#" [ \t]*;
594
de6f8df3b4fe A couple endline fixes in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 584
diff changeset
    80
    NEWLINE = ("\r\n" | "\r" | "\n");
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    81
    WHITESPACE = [ \t\v\f]+;
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    82
*/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    83
679
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    84
    // preprocessor directives are only valid at start of line.
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    85
    if (s->tokenval == ((Token) '\n'))
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    86
        goto ppdirective;  // may jump back to scanner_loop.
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    87
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    88
scanner_loop:
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    89
    if (YYLIMIT == YYCURSOR) YYFILL(1);
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    90
    token = cursor;
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
    91
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    92
/*!re2c
631
5045dcc9fdaf Lexer now handles '\\' continuation at the end of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 616
diff changeset
    93
    "\\" [ \t\v\f]* NEWLINE  { s->line++; goto scanner_loop; }
5045dcc9fdaf Lexer now handles '\\' continuation at the end of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 616
diff changeset
    94
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    95
    "/*"            { goto multilinecomment; }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    96
    "//"            { goto singlelinecomment; }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    97
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    98
    L (L|D)*        { RET(TOKEN_IDENTIFIER); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
    99
    
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   100
    ("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
596
8efa254e8840 Renamed "any" to "ANY" to match other rules.
Ryan C. Gordon <icculus@icculus.org>
parents: 595
diff changeset
   101
    (['] (ESC|ANY\[\r\n\\'])* ['])
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   102
                    { RET(TOKEN_INT_LITERAL); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   103
    
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   104
    (D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   105
                    { RET(TOKEN_FLOAT_LITERAL); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   106
    
596
8efa254e8840 Renamed "any" to "ANY" to match other rules.
Ryan C. Gordon <icculus@icculus.org>
parents: 595
diff changeset
   107
    (["] (ESC|ANY\[\r\n\\"])* ["])
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   108
                    { RET(TOKEN_STRING_LITERAL); }
631
5045dcc9fdaf Lexer now handles '\\' continuation at the end of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 616
diff changeset
   109
584
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   110
    ">>="           { RET(TOKEN_RSHIFTASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   111
    "<<="           { RET(TOKEN_LSHIFTASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   112
    "+="            { RET(TOKEN_ADDASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   113
    "-="            { RET(TOKEN_SUBASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   114
    "*="            { RET(TOKEN_MULTASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   115
    "/="            { RET(TOKEN_DIVASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   116
    "%="            { RET(TOKEN_MODASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   117
    "^="            { RET(TOKEN_XORASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   118
    "&="            { RET(TOKEN_ANDASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   119
    "|="            { RET(TOKEN_ORASSIGN); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   120
    "++"            { RET(TOKEN_INCREMENT); }
9aea8877564d Updated lexer for HLSL.
Ryan C. Gordon <icculus@icculus.org>
parents: 564
diff changeset
   121
    "--"            { RET(TOKEN_DECREMENT); }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   122
    ">>"            { RET(TOKEN_RSHIFT); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   123
    "<<"            { RET(TOKEN_LSHIFT); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   124
    "&&"            { RET(TOKEN_ANDAND); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   125
    "||"            { RET(TOKEN_OROR); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   126
    "<="            { RET(TOKEN_LEQ); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   127
    ">="            { RET(TOKEN_GEQ); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   128
    "=="            { RET(TOKEN_EQL); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   129
    "!="            { RET(TOKEN_NEQ); }
865
4eb06464212d Added stringify operator ("#") to preprocessor.
Ryan C. Gordon <icculus@icculus.org>
parents: 798
diff changeset
   130
    "#"             { RET(TOKEN_HASH); }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   131
    "##"            { RET(TOKEN_HASHHASH); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   132
    "("             { RET('('); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   133
    ")"             { RET(')'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   134
    "["             { RET('['); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   135
    "]"             { RET(']'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   136
    "."             { RET('.'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   137
    ","             { RET(','); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   138
    "&"             { RET('&'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   139
    "!"             { RET('!'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   140
    "~"             { RET('~'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   141
    "-"             { RET('-'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   142
    "+"             { RET('+'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   143
    "*"             { RET('*'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   144
    "/"             { RET('/'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   145
    "%"             { RET('%'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   146
    "<"             { RET('<'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   147
    ">"             { RET('>'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   148
    "^"             { RET('^'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   149
    "|"             { RET('|'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   150
    ":"             { RET(':'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   151
    "{"             { RET('{'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   152
    "}"             { RET('}'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   153
    "="             { RET('='); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   154
    "?"             { RET('?'); }
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   155
690
4710df464f13 Moved asm comment processing into the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 679
diff changeset
   156
    ";"             { if (s->asm_comments) goto singlelinecomment; RET(';'); }
4710df464f13 Moved asm comment processing into the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 679
diff changeset
   157
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   158
    "\000"          { if (eoi) { RET(TOKEN_EOI); } goto bad_chars; }
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   159
635
be3b428802a1 Try to make #error lexing match gcc's.
Ryan C. Gordon <icculus@icculus.org>
parents: 631
diff changeset
   160
    WHITESPACE      { if (s->report_whitespace) RET(' '); goto scanner_loop; }
564
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   161
    NEWLINE         { s->line++; RET('\n'); }
597
832dbfa63509 Bogus chars in the lexer now return a token that signifies this.
Ryan C. Gordon <icculus@icculus.org>
parents: 596
diff changeset
   162
    ANY             { goto bad_chars; }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   163
*/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   164
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   165
multilinecomment:
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   166
    if (YYLIMIT == YYCURSOR) YYFILL(1);
595
a72705c067dc Handle lexing of comment ends better.
Ryan C. Gordon <icculus@icculus.org>
parents: 594
diff changeset
   167
    matchptr = cursor;
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   168
// The "*\/" is just to avoid screwing up text editor syntax highlighting.
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   169
/*!re2c
564
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   170
    "*\/"           {
1121
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   171
                        if (s->report_comments)
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   172
                            RET(TOKEN_MULTI_COMMENT);
635
be3b428802a1 Try to make #error lexing match gcc's.
Ryan C. Gordon <icculus@icculus.org>
parents: 631
diff changeset
   173
                        else if (s->report_whitespace)
be3b428802a1 Try to make #error lexing match gcc's.
Ryan C. Gordon <icculus@icculus.org>
parents: 631
diff changeset
   174
                            RET(' ');
1142
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   175
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   176
                        // Microsoft's preprocessor allows multiline comments
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   177
                        //  before a preprocessor directive, even though C/C++
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   178
                        //  doesn't. See if we've hit this case.
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   179
                        #if MATCH_MICROSOFT_PREPROCESSOR
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   180
                        if (s->tokenval == ((Token) '\n'))  // was start of line?
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   181
                        {
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   182
                            update_state(s, eoi, cursor, token, (Token) '\n');
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   183
                            goto ppdirective;  // may jump back to scanner_loop.
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   184
                        }
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   185
                        #endif
7b554e065321 Allow multiline comments before preprocessor directives.
Ryan C. Gordon <icculus@icculus.org>
parents: 1123
diff changeset
   186
564
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   187
                        goto scanner_loop;
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   188
                    }
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   189
    NEWLINE         {
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   190
                        s->line++;
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   191
                        goto multilinecomment;
c669568326fb Reworked preprocessor newline management.
Ryan C. Gordon <icculus@icculus.org>
parents: 555
diff changeset
   192
                    }
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   193
    "\000"          {
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   194
                        if (eoi)
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   195
                            RET(TOKEN_INCOMPLETE_COMMENT);
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   196
                        goto multilinecomment;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   197
                    }
596
8efa254e8840 Renamed "any" to "ANY" to match other rules.
Ryan C. Gordon <icculus@icculus.org>
parents: 595
diff changeset
   198
    ANY             { goto multilinecomment; }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   199
*/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   200
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   201
singlelinecomment:
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   202
    if (YYLIMIT == YYCURSOR) YYFILL(1);
595
a72705c067dc Handle lexing of comment ends better.
Ryan C. Gordon <icculus@icculus.org>
parents: 594
diff changeset
   203
    matchptr = cursor;
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   204
/*!re2c
1121
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   205
    NEWLINE         {
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   206
                        s->line++;
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   207
                        if (s->report_comments)
1123
3d842e7dd6ad Fix (or just change?) how we report comments vs newlines.
Ryan C. Gordon <icculus@icculus.org>
parents: 1121
diff changeset
   208
                        {
3d842e7dd6ad Fix (or just change?) how we report comments vs newlines.
Ryan C. Gordon <icculus@icculus.org>
parents: 1121
diff changeset
   209
                            cursor = matchptr;  // so we RET('\n') next.
1121
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   210
                            RET(TOKEN_SINGLE_COMMENT);
1123
3d842e7dd6ad Fix (or just change?) how we report comments vs newlines.
Ryan C. Gordon <icculus@icculus.org>
parents: 1121
diff changeset
   211
                        }
1121
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   212
                        token = matchptr;
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   213
                        RET('\n');
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   214
                    }
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   215
    "\000"          {
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   216
                        if (eoi)
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   217
                        {
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   218
                            if (s->report_comments)
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   219
                                RET(TOKEN_SINGLE_COMMENT);
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   220
                            else
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   221
                                RET(TOKEN_EOI);
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   222
                        }
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   223
                        goto singlelinecomment;
4142681f9fda Allow preprocessor to pass through comments, like GNU cpp does.
Ryan C. Gordon <icculus@icculus.org>
parents: 959
diff changeset
   224
                    }
596
8efa254e8840 Renamed "any" to "ANY" to match other rules.
Ryan C. Gordon <icculus@icculus.org>
parents: 595
diff changeset
   225
    ANY             { goto singlelinecomment; }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   226
*/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   227
679
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   228
ppdirective:
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   229
    if (YYLIMIT == YYCURSOR) YYFILL(1);
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   230
/*!re2c
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   231
        PP "include"    { RET(TOKEN_PP_INCLUDE); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   232
        PP "line"       { RET(TOKEN_PP_LINE); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   233
        PP "define"     { RET(TOKEN_PP_DEFINE); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   234
        PP "undef"      { RET(TOKEN_PP_UNDEF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   235
        PP "if"         { RET(TOKEN_PP_IF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   236
        PP "ifdef"      { RET(TOKEN_PP_IFDEF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   237
        PP "ifndef"     { RET(TOKEN_PP_IFNDEF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   238
        PP "else"       { RET(TOKEN_PP_ELSE); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   239
        PP "elif"       { RET(TOKEN_PP_ELIF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   240
        PP "endif"      { RET(TOKEN_PP_ENDIF); }
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   241
        PP "error"      { RET(TOKEN_PP_ERROR); }
910
3ee487d85ecb Handle #pragma (or at least, don't fail on them).
Ryan C. Gordon <icculus@icculus.org>
parents: 865
diff changeset
   242
        PP "pragma"     { RET(TOKEN_PP_PRAGMA); }
679
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   243
        WHITESPACE      { goto ppdirective; }
697
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   244
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   245
        ANY             {
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   246
                            token = cursor = (const uchar *) s->source;
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   247
                            limit = cursor + s->bytes_left;
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   248
                            goto scanner_loop;
765a85a989a2 Fixed crash in lexer on empty input stream.
Ryan C. Gordon <icculus@icculus.org>
parents: 690
diff changeset
   249
                        }
679
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   250
*/
3892ebde7d99 Fixed lexer to only accept preprocessor directives at start of a line.
Ryan C. Gordon <icculus@icculus.org>
parents: 651
diff changeset
   251
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   252
bad_chars:
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   253
    if (YYLIMIT == YYCURSOR) YYFILL(1);
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   254
/*!re2c
601
bb1484be4e1b Reworked and documented preprocessor tokens.
Ryan C. Gordon <icculus@icculus.org>
parents: 597
diff changeset
   255
    ANYLEGAL        { cursor--; RET(TOKEN_BAD_CHARS); }
642
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   256
    "\000"          {
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   257
                        if (eoi)
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   258
                        {
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   259
                            assert( !((token >= sentinel) &&
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   260
                                     (token < sentinel+YYMAXFILL)) );
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   261
                            eoi = 0;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   262
                            cursor = (uchar *) s->source_base + s->orig_length;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   263
                            RET(TOKEN_BAD_CHARS);  // next call will be EOI.
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   264
                        }
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   265
                        goto bad_chars;
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   266
                    }
624ab8696f9b Enormous amount of tapdancing to handle EOI better in the lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 635
diff changeset
   267
597
832dbfa63509 Bogus chars in the lexer now return a token that signifies this.
Ryan C. Gordon <icculus@icculus.org>
parents: 596
diff changeset
   268
    ANY             { goto bad_chars; }
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   269
*/
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   270
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   271
    assert(0 && "Shouldn't hit this code");
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   272
    RET(TOKEN_UNKNOWN);
649
527cfe6f2d45 Renamed preprocessor_internal_lexer to preprocessor_lexer.
Ryan C. Gordon <icculus@icculus.org>
parents: 648
diff changeset
   273
} // preprocessor_lexer
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   274
1147
ff14a55da268 Fixed a few outdated comments.
Ryan C. Gordon <icculus@icculus.org>
parents: 1142
diff changeset
   275
// end of mojoshader_lexer.re (or .c) ...
555
940821555fda Initial work on preprocessor. Not yet complete!
Ryan C. Gordon <icculus@icculus.org>
parents:
diff changeset
   276