2 ** $Id: llex.c,v 2.63.1.2 2013/08/30 15:49:41 roberto Exp $
4 ** See Copyright Notice in lua.h
28 #define next(ls) (ls->current = zgetc(ls->z))
32 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
36 static const char *const luaX_tokens [] = {
37 "and", "break", "do", "else", "elseif",
38 "end", "false", "for", "function", "goto", "if",
39 "in", "local", "nil", "not", "or", "repeat",
40 "return", "then", "true", "until", "while",
41 "..", "...", "==", ">=", "<=", "~=", "::", "<eof>",
42 "<number>", "<name>", "<string>"
46 #define save_and_next(ls) (save(ls, ls->current), next(ls))
49 static l_noret lexerror (LexState *ls, const char *msg, int token);
52 static void save (LexState *ls, int c) {
53 Mbuffer *b = ls->buff;
54 if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
56 if (luaZ_sizebuffer(b) >= MAX_SIZET/2)
57 lexerror(ls, "lexical element too long", 0);
58 newsize = luaZ_sizebuffer(b) * 2;
59 luaZ_resizebuffer(ls->L, b, newsize);
61 b->buffer[luaZ_bufflen(b)++] = cast(char, c);
65 void luaX_init (lua_State *L) {
67 for (i=0; i<NUM_RESERVED; i++) {
68 TString *ts = luaS_new(L, luaX_tokens[i]);
69 luaS_fix(ts); /* reserved words are never collected */
70 ts->tsv.extra = cast_byte(i+1); /* reserved word */
75 const char *luaX_token2str (LexState *ls, int token) {
76 if (token < FIRST_RESERVED) { /* single-byte symbols? */
77 lua_assert(token == cast(unsigned char, token));
78 return (lisprint(token)) ? luaO_pushfstring(ls->L, LUA_QL("%c"), token) :
79 luaO_pushfstring(ls->L, "char(%d)", token);
82 const char *s = luaX_tokens[token - FIRST_RESERVED];
83 if (token < TK_EOS) /* fixed format (symbols and reserved words)? */
84 return luaO_pushfstring(ls->L, LUA_QS, s);
85 else /* names, strings, and numerals */
91 static const char *txtToken (LexState *ls, int token) {
97 return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff));
99 return luaX_token2str(ls, token);
104 static l_noret lexerror (LexState *ls, const char *msg, int token) {
105 char buff[LUA_IDSIZE];
106 luaO_chunkid(buff, getstr(ls->source), LUA_IDSIZE);
107 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
109 luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
110 luaD_throw(ls->L, LUA_ERRSYNTAX);
114 l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
115 lexerror(ls, msg, ls->t.token);
120 ** creates a new string and anchors it in function's table so that
121 ** it will not be collected until the end of the function's compilation
122 ** (by that time it should be anchored in function's prototype)
124 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
125 lua_State *L = ls->L;
126 TValue *o; /* entry for `str' */
127 TString *ts = luaS_newlstr(L, str, l); /* create new string */
128 setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
129 o = luaH_set(L, ls->fs->h, L->top - 1);
130 if (ttisnil(o)) { /* not in use yet? (see 'addK') */
131 /* boolean value does not need GC barrier;
132 table has no metatable, so it does not need to invalidate cache */
133 setbvalue(o, 1); /* t[string] = true */
136 else { /* string already present */
137 ts = rawtsvalue(keyfromval(o)); /* re-use value previously stored */
139 L->top--; /* remove string from stack */
145 ** increment line number and skips newline sequence (any of
146 ** \n, \r, \n\r, or \r\n)
148 static void inclinenumber (LexState *ls) {
149 int old = ls->current;
150 lua_assert(currIsNewline(ls));
151 next(ls); /* skip `\n' or `\r' */
152 if (currIsNewline(ls) && ls->current != old)
153 next(ls); /* skip `\n\r' or `\r\n' */
154 if (++ls->linenumber >= MAX_INT)
155 luaX_syntaxerror(ls, "chunk has too many lines");
159 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
163 ls->current = firstchar;
164 ls->lookahead.token = TK_EOS; /* no look-ahead token */
170 ls->envn = luaS_new(L, LUA_ENV); /* create env name */
171 luaS_fix(ls->envn); /* never collect this name */
172 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
178 ** =======================================================
180 ** =======================================================
185 static int check_next (LexState *ls, const char *set) {
186 if (ls->current == '\0' || !strchr(set, ls->current))
194 ** change all characters 'from' in buffer to 'to'
196 static void buffreplace (LexState *ls, char from, char to) {
197 size_t n = luaZ_bufflen(ls->buff);
198 char *p = luaZ_buffer(ls->buff);
200 if (p[n] == from) p[n] = to;
204 #if !defined(getlocaledecpoint)
205 #define getlocaledecpoint() (localeconv()->decimal_point[0])
209 #define buff2d(b,e) luaO_str2d(luaZ_buffer(b), luaZ_bufflen(b) - 1, e)
212 ** in case of format error, try to change decimal point separator to
213 ** the one defined in the current locale and check again
215 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
216 char old = ls->decpoint;
217 ls->decpoint = getlocaledecpoint();
218 buffreplace(ls, old, ls->decpoint); /* try new decimal separator */
219 if (!buff2d(ls->buff, &seminfo->r)) {
220 /* format error with correct decimal point: no more options */
221 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */
222 lexerror(ls, "malformed number", TK_NUMBER);
229 ** this function is quite liberal in what it accepts, as 'luaO_str2d'
230 ** will reject ill-formed numerals.
232 static void read_numeral (LexState *ls, SemInfo *seminfo) {
233 const char *expo = "Ee";
234 int first = ls->current;
235 lua_assert(lisdigit(ls->current));
237 if (first == '0' && check_next(ls, "Xx")) /* hexadecimal? */
240 if (check_next(ls, expo)) /* exponent part? */
241 check_next(ls, "+-"); /* optional exponent sign */
242 if (lisxdigit(ls->current) || ls->current == '.')
247 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */
248 if (!buff2d(ls->buff, &seminfo->r)) /* format error? */
249 trydecpoint(ls, seminfo); /* try to update decimal point separator */
254 ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or
255 ** -1 if sequence is malformed
257 static int skip_sep (LexState *ls) {
260 lua_assert(s == '[' || s == ']');
262 while (ls->current == '=') {
266 return (ls->current == s) ? count : (-count) - 1;
270 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
271 save_and_next(ls); /* skip 2nd `[' */
272 if (currIsNewline(ls)) /* string starts with a newline? */
273 inclinenumber(ls); /* skip it */
275 switch (ls->current) {
277 lexerror(ls, (seminfo) ? "unfinished long string" :
278 "unfinished long comment", TK_EOS);
279 break; /* to avoid warnings */
281 if (skip_sep(ls) == sep) {
282 save_and_next(ls); /* skip 2nd `]' */
287 case '\n': case '\r': {
290 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
294 if (seminfo) save_and_next(ls);
300 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
301 luaZ_bufflen(ls->buff) - 2*(2 + sep));
305 static void escerror (LexState *ls, int *c, int n, const char *msg) {
307 luaZ_resetbuffer(ls->buff); /* prepare error message */
309 for (i = 0; i < n && c[i] != EOZ; i++)
311 lexerror(ls, msg, TK_STRING);
315 static int readhexaesc (LexState *ls) {
316 int c[3], i; /* keep input for error message */
317 int r = 0; /* result accumulator */
318 c[0] = 'x'; /* for error message */
319 for (i = 1; i < 3; i++) { /* read two hexadecimal digits */
321 if (!lisxdigit(c[i]))
322 escerror(ls, c, i + 1, "hexadecimal digit expected");
323 r = (r << 4) + luaO_hexavalue(c[i]);
329 static int readdecesc (LexState *ls) {
331 int r = 0; /* result accumulator */
332 for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */
334 r = 10*r + c[i] - '0';
338 escerror(ls, c, i, "decimal escape too large");
343 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
344 save_and_next(ls); /* keep delimiter (for error messages) */
345 while (ls->current != del) {
346 switch (ls->current) {
348 lexerror(ls, "unfinished string", TK_EOS);
349 break; /* to avoid warnings */
352 lexerror(ls, "unfinished string", TK_STRING);
353 break; /* to avoid warnings */
354 case '\\': { /* escape sequences */
355 int c; /* final character to be saved */
356 next(ls); /* do not save the `\' */
357 switch (ls->current) {
358 case 'a': c = '\a'; goto read_save;
359 case 'b': c = '\b'; goto read_save;
360 case 'f': c = '\f'; goto read_save;
361 case 'n': c = '\n'; goto read_save;
362 case 'r': c = '\r'; goto read_save;
363 case 't': c = '\t'; goto read_save;
364 case 'v': c = '\v'; goto read_save;
365 case 'x': c = readhexaesc(ls); goto read_save;
366 case '\n': case '\r':
367 inclinenumber(ls); c = '\n'; goto only_save;
368 case '\\': case '\"': case '\'':
369 c = ls->current; goto read_save;
370 case EOZ: goto no_save; /* will raise an error next loop */
371 case 'z': { /* zap following span of spaces */
372 next(ls); /* skip the 'z' */
373 while (lisspace(ls->current)) {
374 if (currIsNewline(ls)) inclinenumber(ls);
380 if (!lisdigit(ls->current))
381 escerror(ls, &ls->current, 1, "invalid escape sequence");
382 /* digital escape \ddd */
387 read_save: next(ls); /* read next character */
388 only_save: save(ls, c); /* save 'c' */
395 save_and_next(ls); /* skip delimiter */
396 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
397 luaZ_bufflen(ls->buff) - 2);
401 static int llex (LexState *ls, SemInfo *seminfo) {
402 luaZ_resetbuffer(ls->buff);
404 switch (ls->current) {
405 case '\n': case '\r': { /* line breaks */
409 case ' ': case '\f': case '\t': case '\v': { /* spaces */
413 case '-': { /* '-' or '--' (comment) */
415 if (ls->current != '-') return '-';
416 /* else is a comment */
418 if (ls->current == '[') { /* long comment? */
419 int sep = skip_sep(ls);
420 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */
422 read_long_string(ls, NULL, sep); /* skip long comment */
423 luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */
427 /* else short comment */
428 while (!currIsNewline(ls) && ls->current != EOZ)
429 next(ls); /* skip until end of line (or end of file) */
432 case '[': { /* long string or simply '[' */
433 int sep = skip_sep(ls);
435 read_long_string(ls, seminfo, sep);
438 else if (sep == -1) return '[';
439 else lexerror(ls, "invalid long string delimiter", TK_STRING);
443 if (ls->current != '=') return '=';
444 else { next(ls); return TK_EQ; }
448 if (ls->current != '=') return '<';
449 else { next(ls); return TK_LE; }
453 if (ls->current != '=') return '>';
454 else { next(ls); return TK_GE; }
458 if (ls->current != '=') return '~';
459 else { next(ls); return TK_NE; }
463 if (ls->current != ':') return ':';
464 else { next(ls); return TK_DBCOLON; }
466 case '"': case '\'': { /* short literal strings */
467 read_string(ls, ls->current, seminfo);
470 case '.': { /* '.', '..', '...', or number */
472 if (check_next(ls, ".")) {
473 if (check_next(ls, "."))
474 return TK_DOTS; /* '...' */
475 else return TK_CONCAT; /* '..' */
477 else if (!lisdigit(ls->current)) return '.';
478 /* else go through */
480 case '0': case '1': case '2': case '3': case '4':
481 case '5': case '6': case '7': case '8': case '9': {
482 read_numeral(ls, seminfo);
489 if (lislalpha(ls->current)) { /* identifier or reserved word? */
493 } while (lislalnum(ls->current));
494 ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
495 luaZ_bufflen(ls->buff));
497 if (isreserved(ts)) /* reserved word? */
498 return ts->tsv.extra - 1 + FIRST_RESERVED;
503 else { /* single-char tokens (+ - / ...) */
514 void luaX_next (LexState *ls) {
515 ls->lastline = ls->linenumber;
516 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
517 ls->t = ls->lookahead; /* use this one */
518 ls->lookahead.token = TK_EOS; /* and discharge it */
521 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
525 int luaX_lookahead (LexState *ls) {
526 lua_assert(ls->lookahead.token == TK_EOS);
527 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
528 return ls->lookahead.token;