mojoshader_lexer.re
1
2
3
4
5
6
7
8
9
10
11
12
/**
* MojoShader; generate shader programs from bytecode of compiled
* Direct3D shaders.
*
* Please see the file LICENSE.txt in the source's root directory.
*
* This file written by Ryan C. Gordon.
*/
// This was originally based on examples/pp-c.re from re2c: http://re2c.org/
// re2c is public domain code.
//
13
14
// You build mojoshader_lexer.c from the .re file with re2c...
// re2c -is -o mojoshader_lexer.c mojoshader_lexer.re
15
16
17
18
19
20
21
22
23
24
25
26
//
// Changes to the lexer are done to the .re file, not the C code!
//
// Please note that this isn't a perfect C lexer, since it is used for both
// HLSL and shader assembly language, and follows the quirks of Microsoft's
// tools.
#define __MOJOSHADER_INTERNAL__ 1
#include "mojoshader_internal.h"
typedef unsigned char uchar;
27
/*!max:re2c */
28
#define RET(t) return update_state(s, eoi, cursor, token, (Token) t)
29
30
31
32
#define YYCTYPE uchar
#define YYCURSOR cursor
#define YYLIMIT limit
#define YYMARKER s->lexer_marker
33
#define YYFILL(n) { if ((n) == 1) { cursor = sentinel; limit = cursor + YYMAXFILL; eoi = 1; } }
35
36
static uchar sentinel[YYMAXFILL];
37
38
static Token update_state(IncludeState *s, int eoi, const uchar *cur,
const uchar *tok, const Token val)
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
if (eoi)
{
s->bytes_left = 0;
s->source = (const char *) s->source_base + s->orig_length;
if ( (tok >= sentinel) && (tok < (sentinel+YYMAXFILL)) )
s->token = s->source;
else
s->token = (const char *) tok;
} // if
else
{
s->bytes_left -= (unsigned int) (cur - ((const uchar *) s->source));
s->source = (const char *) cur;
s->token = (const char *) tok;
} // else
55
s->tokenlen = (unsigned int) (s->source - s->token);
56
57
s->tokenval = val;
return val;
58
59
} // update_state
60
Token preprocessor_lexer(IncludeState *s)
61
62
{
const uchar *cursor = (const uchar *) s->source;
63
const uchar *token = cursor;
64
const uchar *matchptr;
65
const uchar *limit = cursor + s->bytes_left;
67
68
/*!re2c
69
ANY = [\000-\377];
70
ANYLEGAL = [a-zA-Z0-9_/'*=+%^&|!#<>()[{}.,~^:;? \t\v\f\r\n\-\]\\];
71
72
73
74
75
O = [0-7];
D = [0-9];
L = [a-zA-Z_];
H = [a-fA-F0-9];
E = [Ee] [+-]? D+;
76
FS = [fFhH];
77
78
79
IS = [uUlL]*;
ESC = [\\] ([abfnrtv?'"\\] | "x" H+ | O+);
PP = "#" [ \t]*;
80
NEWLINE = ("\r\n" | "\r" | "\n");
81
82
83
WHITESPACE = [ \t\v\f]+;
*/
84
85
86
87
88
89
90
91
// preprocessor directives are only valid at start of line.
if (s->tokenval == ((Token) '\n'))
goto ppdirective; // may jump back to scanner_loop.
scanner_loop:
if (YYLIMIT == YYCURSOR) YYFILL(1);
token = cursor;
93
94
"\\" [ \t\v\f]* NEWLINE { s->line++; goto scanner_loop; }
95
96
97
98
99
100
"/*" { goto multilinecomment; }
"//" { goto singlelinecomment; }
L (L|D)* { RET(TOKEN_IDENTIFIER); }
("0" [xX] H+ IS?) | ("0" D+ IS?) | (D+ IS?) |
101
(['] (ESC|ANY\[\r\n\\'])* ['])
102
103
104
105
106
{ RET(TOKEN_INT_LITERAL); }
(D+ E FS?) | (D* "." D+ E? FS?) | (D+ "." D* E? FS?)
{ RET(TOKEN_FLOAT_LITERAL); }
107
(["] (ESC|ANY\[\r\n\\"])* ["])
108
{ RET(TOKEN_STRING_LITERAL); }
110
111
112
113
114
115
116
117
118
119
120
121
">>=" { RET(TOKEN_RSHIFTASSIGN); }
"<<=" { RET(TOKEN_LSHIFTASSIGN); }
"+=" { RET(TOKEN_ADDASSIGN); }
"-=" { RET(TOKEN_SUBASSIGN); }
"*=" { RET(TOKEN_MULTASSIGN); }
"/=" { RET(TOKEN_DIVASSIGN); }
"%=" { RET(TOKEN_MODASSIGN); }
"^=" { RET(TOKEN_XORASSIGN); }
"&=" { RET(TOKEN_ANDASSIGN); }
"|=" { RET(TOKEN_ORASSIGN); }
"++" { RET(TOKEN_INCREMENT); }
"--" { RET(TOKEN_DECREMENT); }
122
123
124
125
126
127
128
129
">>" { RET(TOKEN_RSHIFT); }
"<<" { RET(TOKEN_LSHIFT); }
"&&" { RET(TOKEN_ANDAND); }
"||" { RET(TOKEN_OROR); }
"<=" { RET(TOKEN_LEQ); }
">=" { RET(TOKEN_GEQ); }
"==" { RET(TOKEN_EQL); }
"!=" { RET(TOKEN_NEQ); }
130
"#" { RET(TOKEN_HASH); }
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
"##" { RET(TOKEN_HASHHASH); }
"(" { RET('('); }
")" { RET(')'); }
"[" { RET('['); }
"]" { RET(']'); }
"." { RET('.'); }
"," { RET(','); }
"&" { RET('&'); }
"!" { RET('!'); }
"~" { RET('~'); }
"-" { RET('-'); }
"+" { RET('+'); }
"*" { RET('*'); }
"/" { RET('/'); }
"%" { RET('%'); }
"<" { RET('<'); }
">" { RET('>'); }
"^" { RET('^'); }
"|" { RET('|'); }
":" { RET(':'); }
"{" { RET('{'); }
"}" { RET('}'); }
"=" { RET('='); }
"?" { RET('?'); }
156
157
";" { if (s->asm_comments) goto singlelinecomment; RET(';'); }
158
159
"\000" { if (eoi) { RET(TOKEN_EOI); } goto bad_chars; }
160
WHITESPACE { if (s->report_whitespace) RET(' '); goto scanner_loop; }
161
NEWLINE { s->line++; RET('\n'); }
162
ANY { goto bad_chars; }
163
164
165
*/
multilinecomment:
166
if (YYLIMIT == YYCURSOR) YYFILL(1);
167
matchptr = cursor;
168
169
// The "*\/" is just to avoid screwing up text editor syntax highlighting.
/*!re2c
170
"*\/" {
171
172
if (s->report_comments)
RET(TOKEN_MULTI_COMMENT);
173
174
else if (s->report_whitespace)
RET(' ');
175
176
177
178
179
180
181
182
183
184
185
186
// Microsoft's preprocessor allows multiline comments
// before a preprocessor directive, even though C/C++
// doesn't. See if we've hit this case.
#if MATCH_MICROSOFT_PREPROCESSOR
if (s->tokenval == ((Token) '\n')) // was start of line?
{
update_state(s, eoi, cursor, token, (Token) '\n');
goto ppdirective; // may jump back to scanner_loop.
}
#endif
187
188
189
190
191
192
goto scanner_loop;
}
NEWLINE {
s->line++;
goto multilinecomment;
}
193
194
195
196
197
"\000" {
if (eoi)
RET(TOKEN_INCOMPLETE_COMMENT);
goto multilinecomment;
}
198
ANY { goto multilinecomment; }
199
200
201
*/
singlelinecomment:
202
if (YYLIMIT == YYCURSOR) YYFILL(1);
203
matchptr = cursor;
205
206
207
NEWLINE {
s->line++;
if (s->report_comments)
208
209
{
cursor = matchptr; // so we RET('\n') next.
210
RET(TOKEN_SINGLE_COMMENT);
212
213
214
215
216
217
218
219
220
221
222
223
224
token = matchptr;
RET('\n');
}
"\000" {
if (eoi)
{
if (s->report_comments)
RET(TOKEN_SINGLE_COMMENT);
else
RET(TOKEN_EOI);
}
goto singlelinecomment;
}
225
ANY { goto singlelinecomment; }
228
229
230
231
232
233
234
235
236
237
238
239
240
241
ppdirective:
if (YYLIMIT == YYCURSOR) YYFILL(1);
/*!re2c
PP "include" { RET(TOKEN_PP_INCLUDE); }
PP "line" { RET(TOKEN_PP_LINE); }
PP "define" { RET(TOKEN_PP_DEFINE); }
PP "undef" { RET(TOKEN_PP_UNDEF); }
PP "if" { RET(TOKEN_PP_IF); }
PP "ifdef" { RET(TOKEN_PP_IFDEF); }
PP "ifndef" { RET(TOKEN_PP_IFNDEF); }
PP "else" { RET(TOKEN_PP_ELSE); }
PP "elif" { RET(TOKEN_PP_ELIF); }
PP "endif" { RET(TOKEN_PP_ENDIF); }
PP "error" { RET(TOKEN_PP_ERROR); }
242
PP "pragma" { RET(TOKEN_PP_PRAGMA); }
243
WHITESPACE { goto ppdirective; }
244
245
246
247
248
249
ANY {
token = cursor = (const uchar *) s->source;
limit = cursor + s->bytes_left;
goto scanner_loop;
}
252
bad_chars:
253
if (YYLIMIT == YYCURSOR) YYFILL(1);
255
ANYLEGAL { cursor--; RET(TOKEN_BAD_CHARS); }
256
257
258
259
260
261
262
263
264
265
266
267
"\000" {
if (eoi)
{
assert( !((token >= sentinel) &&
(token < sentinel+YYMAXFILL)) );
eoi = 0;
cursor = (uchar *) s->source_base + s->orig_length;
RET(TOKEN_BAD_CHARS); // next call will be EOI.
}
goto bad_chars;
}
268
ANY { goto bad_chars; }
269
270
271
272
*/
assert(0 && "Shouldn't hit this code");
RET(TOKEN_UNKNOWN);
273
} // preprocessor_lexer
275
// end of mojoshader_lexer.re (or .c) ...