mojoshader_lexer.re
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/**
* MojoShader; generate shader programs from bytecode of compiled
* Direct3D shaders.
*
* Please see the file LICENSE.txt in the source's root directory.
*
* This file written by Ryan C. Gordon.
*/
// This was originally based on examples/pp-c.re from re2c: http://re2c.org/
// re2c is public domain code.
//
// You build mojoshader_lexer_preprocessor.c from the .re file with re2c...
// re2c -is -o mojoshader_lexer_preprocessor.c mojoshader_lexer_preprocessor.re
//
// Changes to the lexer are done to the .re file, not the C code!
//
// Please note that this isn't a perfect C lexer, since it is used for both
// HLSL and shader assembly language, and follows the quirks of Microsoft's
// tools.
#define __MOJOSHADER_INTERNAL__ 1
#include "mojoshader_internal.h"
typedef unsigned char uchar;
27
/*!max:re2c */
28
#define RET(t) return update_state(s, eoi, cursor, token, (Token) t)
29
30
31
32
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT limit
#define YYMARKER s->lexer_marker
33
#define YYFILL(n) { if ((n) == 1) { cursor = sentinel; limit = cursor + YYMAXFILL; eoi = 1; } }
35
36
static uchar sentinel[YYMAXFILL];
37
38
static Token update_state(IncludeState *s, int eoi, const uchar *cur,
const uchar *tok, const Token val)
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
if (eoi)
{
s->bytes_left = 0;
s->source = (const char *) s->source_base + s->orig_length;
if ( (tok >= sentinel) && (tok < (sentinel+YYMAXFILL)) )
s->token = s->source;
else
s->token = (const char *) tok;
} // if
else
{
s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
s->source = (const char *) cur;
s->token = (const char *) tok;
} // else
55
s->tokenlen = (unsigned int) (s->source - s->token);
56
57
s->tokenval = val;
return val;
58
59
} // update_state
60
Token preprocessor_lexer(IncludeState *s)
61
62
{
const uchar *cursor = (const uchar *) s->source;
63
const uchar *token = cursor;
64
const uchar *matchptr;
65
const uchar *limit = cursor + s->bytes_left;
67
int saw_newline = 0;
68
69
/*!re2c
70
ANY = [\000-\377];
71
ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
72
73
74
75
76
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
77
FS = [fFhH];
78
79
80
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
PP = "#" [ \t]*;
81
NEWLINE = ("\r\n" | "\r" | "\n");
82
83
84
WHITESPACE = [ \t\v\f]+;
*/
85
86
87
88
89
90
91
92
// preprocessor directives are only valid at start of line.
if (s->tokenval == ((Token) '\n'))
goto ppdirective; // may jump back to scanner_loop.
scanner_loop:
if (YYLIMIT == YYCURSOR) YYFILL(1);
token = cursor;
94
95
"\\" [ \t\v\f]* NEWLINE { s->line++; goto scanner_loop; }
96
97
98
99
100
101
"/*" { goto multilinecomment; }
"//" { goto singlelinecomment; }
L (L|D)* { RET(TOKEN_IDENTIFIER); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
102
(['] (ESC|ANY\[\r\n\\'])* ['])
103
104
105
106
107
{ RET(TOKEN_INT_LITERAL); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(TOKEN_FLOAT_LITERAL); }
108
(["] (ESC|ANY\[\r\n\\"])* ["])
109
{ RET(TOKEN_STRING_LITERAL); }
111
112
113
114
115
116
117
118
119
120
121
122
">>=" { RET(TOKEN_RSHIFTASSIGN); }
"<<=" { RET(TOKEN_LSHIFTASSIGN); }
"+=" { RET(TOKEN_ADDASSIGN); }
"-=" { RET(TOKEN_SUBASSIGN); }
"*=" { RET(TOKEN_MULTASSIGN); }
"/=" { RET(TOKEN_DIVASSIGN); }
"%=" { RET(TOKEN_MODASSIGN); }
"^=" { RET(TOKEN_XORASSIGN); }
"&=" { RET(TOKEN_ANDASSIGN); }
"|=" { RET(TOKEN_ORASSIGN); }
"++" { RET(TOKEN_INCREMENT); }
"--" { RET(TOKEN_DECREMENT); }
123
124
125
126
127
128
129
130
">>" { RET(TOKEN_RSHIFT); }
"<<" { RET(TOKEN_LSHIFT); }
"&&" { RET(TOKEN_ANDAND); }
"||" { RET(TOKEN_OROR); }
"<=" { RET(TOKEN_LEQ); }
">=" { RET(TOKEN_GEQ); }
"==" { RET(TOKEN_EQL); }
"!=" { RET(TOKEN_NEQ); }
131
"#" { RET(TOKEN_HASH); }
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
"##" { RET(TOKEN_HASHHASH); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"," { RET(','); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
":" { RET(':'); }
"{" { RET('{'); }
"}" { RET('}'); }
"=" { RET('='); }
"?" { RET('?'); }
157
158
";" { if (s->asm_comments) goto singlelinecomment; RET(';'); }
159
160
"\000" { if (eoi) { RET(TOKEN_EOI); } goto bad_chars; }
161
WHITESPACE { if (s->report_whitespace) RET(' '); goto scanner_loop; }
162
NEWLINE { s->line++; RET('\n'); }
163
ANY { goto bad_chars; }
164
165
166
*/
multilinecomment:
167
if (YYLIMIT == YYCURSOR) YYFILL(1);
168
matchptr = cursor;
169
170
// The "*\/" is just to avoid screwing up text editor syntax highlighting.
/*!re2c
171
172
173
"*\/" {
if (saw_newline)
RET('\n');
174
175
else if (s->report_whitespace)
RET(' ');
176
177
178
179
goto scanner_loop;
}
NEWLINE {
s->line++;
180
token = matchptr;
181
182
183
saw_newline = 1;
goto multilinecomment;
}
184
185
186
187
188
"\000" {
if (eoi)
RET(TOKEN_INCOMPLETE_COMMENT);
goto multilinecomment;
}
189
ANY { goto multilinecomment; }
190
191
192
*/
singlelinecomment:
193
if (YYLIMIT == YYCURSOR) YYFILL(1);
194
matchptr = cursor;
196
NEWLINE { s->line++; token = matchptr; RET('\n'); }
197
"\000" { if (eoi) { RET(TOKEN_EOI); } goto singlelinecomment; }
198
ANY { goto singlelinecomment; }
201
202
203
204
205
206
207
208
209
210
211
212
213
214
ppdirective:
if (YYLIMIT == YYCURSOR) YYFILL(1);
/*!re2c
PP "include" { RET(TOKEN_PP_INCLUDE); }
PP "line" { RET(TOKEN_PP_LINE); }
PP "define" { RET(TOKEN_PP_DEFINE); }
PP "undef" { RET(TOKEN_PP_UNDEF); }
PP "if" { RET(TOKEN_PP_IF); }
PP "ifdef" { RET(TOKEN_PP_IFDEF); }
PP "ifndef" { RET(TOKEN_PP_IFNDEF); }
PP "else" { RET(TOKEN_PP_ELSE); }
PP "elif" { RET(TOKEN_PP_ELIF); }
PP "endif" { RET(TOKEN_PP_ENDIF); }
PP "error" { RET(TOKEN_PP_ERROR); }
215
PP "pragma" { RET(TOKEN_PP_PRAGMA); }
216
WHITESPACE { goto ppdirective; }
217
218
219
220
221
222
ANY {
token = cursor = (const uchar *) s->source;
limit = cursor + s->bytes_left;
goto scanner_loop;
}
225
bad_chars:
226
if (YYLIMIT == YYCURSOR) YYFILL(1);
228
ANYLEGAL { cursor--; RET(TOKEN_BAD_CHARS); }
229
230
231
232
233
234
235
236
237
238
239
240
"\000" {
if (eoi)
{
assert( !((token >= sentinel) &&
(token < sentinel+YYMAXFILL)) );
eoi = 0;
cursor = (uchar *) s->source_base + s->orig_length;
RET(TOKEN_BAD_CHARS); // next call will be EOI.
}
goto bad_chars;
}
241
ANY { goto bad_chars; }
242
243
244
245
*/
assert(0 && "Shouldn't hit this code");
RET(TOKEN_UNKNOWN);
246
} // preprocessor_lexer
247
248
// end of mojoshader_lexer_preprocessor.re (or .c) ...