commit 1ae4a875f9b9215126fd406329cbfd267271492d
parent 3f1b9619b57cbd1e479f281c86f8ed3fabe2b024
Author: Roberto E. Vargas Caballero <k0ga@shike2.com>
Date: Sat, 9 May 2015 17:01:22 +0200
Rewrite lexer analizer
This new lexer uses the full line as input, and it does not read
character by character.
Diffstat:
| M | cc1/cc1.h | | | 1 | - |
| M | cc1/lex.c | | | 552 | +++++++++++++++++++++++++++++++++---------------------------------------------- |
| M | cc1/symbol.c | | | 38 | ++++++-------------------------------- |
3 files changed, 238 insertions(+), 353 deletions(-)
diff --git a/cc1/cc1.h b/cc1/cc1.h
@@ -40,7 +40,6 @@ struct symbol {
int i;
char *s;
uint8_t token;
- void (*fun)(char *);
} u;
struct symbol *next;
struct symbol *hash;
diff --git a/cc1/lex.c b/cc1/lex.c
@@ -1,5 +1,6 @@
#include <assert.h>
+#include <errno.h>
#include <inttypes.h>
#include <setjmp.h>
#include <stdio.h>
@@ -18,21 +19,17 @@ typedef struct input Input;
struct input {
char *fname;
unsigned short nline;
- int cnt;
FILE *fp;
- char *line, *ptr;
+ char *line, *begin, *p;
struct input *next;
};
-#define nextchar() ((--input->cnt >= 0) ? \
- (unsigned char) *input->ptr++ : readline())
-
uint8_t lex_ns = NS_IDEN;
uint8_t yytoken;
struct yystype yylval;
char yytext[IDENTSIZ + 1];
-static uint8_t safe, comment, commentline;
+static uint8_t safe;
static Input *input;
bool
@@ -59,7 +56,6 @@ addinput(char *fname)
ip->fname = fname;
ip->next = input;
ip->line = NULL;
- ip->cnt = 0;
ip->nline = nline;
ip->fp = fp;
input = ip;
@@ -95,53 +91,13 @@ fileline(void)
return input->nline;
}
-static void
-newline(void)
-{
- if (++input->nline == 0)
- die("input file '%s' too long", input->fname);
-}
-
/* TODO: preprocessor error must not rise recover */
-static void
-preprocessor(void)
-{
- char str[IDENTSIZ+1], *p, *q;
- unsigned short cnt, n;
- Symbol *sym;
-
- p = input->ptr;
- q = &p[input->cnt-1];
- while (q > p && isspace(*q))
- ++q;
- while (isspace(*p))
- ++p;
- for (q = p; isalpha(*q); ++q)
- /* nothing */;
- if ((n = q - p) > IDENTSIZ)
- goto bad_directive;
- strncpy(str, p, n);
- str[n] = '\0';
-
- /* discard this line for the lexer */
- input->cnt = 0;
- if ((sym = lookup(str, NS_CPP)) == NULL)
- goto bad_directive;
- (*sym->u.fun)(q);
- return;
-
-bad_directive:
- error("incorrect preprocessor directive");
-}
-
-void
+char *
include(char *s)
{
char fname[FILENAME_MAX], delim, c, *p;
size_t len;
- while (isspace(*s))
- ++s;
if ((c = *s++) == '>')
delim = '>';
else if (c == '"')
@@ -162,11 +118,11 @@ include(char *s)
fname[len] = '\0';
if (!addinput(fname))
goto not_found;
- return;
+ } else {
+ abort();
}
- abort();
- return;
+ return p+1;
not_found:
error("included file '%s' not found", fname);
@@ -176,34 +132,41 @@ bad_include:
error("#include expects \"FILENAME\" or <FILENAME>");
}
-void
-define(char *str)
-{
-
-}
-
-void
-undef(char *str)
-{
- fprintf(stderr, "Esto en un undef\n");
-}
-
-void
-ifdef(char *str)
-{
- fprintf(stderr, "Esto en un ifdef\n");
-}
-
-void
-ifndef(char *str)
-{
- fprintf(stderr, "Esto en un ifndef\n");
-}
+static char *
+preprocessor(char *p)
+{
+ char str[IDENTSIZ+1], *q;
+ unsigned short n;
+ static char **bp, *cmds[] = {
+ "include",
+ NULL
+ };
+ static char *(*funs[])(char *) = {
+ include
+ };
-void
-endif(char *str)
-{
- fprintf(stderr, "Esto en un endif\n");
+ while (isspace(*p))
+ ++p;
+ if (*p != '#')
+ return p;
+ for (++p; isspace(*p); ++p)
+ /* nothing */;
+ for (q = p; isalpha(*q); ++q)
+ /* nothing */;
+ n = q - p;
+ while (isspace(*q))
+ ++q;
+ for (bp = cmds; *bp; ++bp) {
+ if (strncmp(*bp, p, n))
+ continue;
+ q = (*funs[bp - cmds])(q);
+ while (isspace(*q++))
+ /* nothing */;
+ if (*q != '\0')
+ error("trailing characters after preprocessor directive");
+ return NULL;
+ }
+ error("incorrect preprocessor directive");
}
static int
@@ -222,31 +185,15 @@ repeat:
return c;
}
-static int
+static void
readline(void)
{
- char *bp, *ptr;
- uint8_t n;
+ static int comment, commentline;
+ char *bp, *lim;
int c;
- FILE *fp;
-repeat:
- if (!input)
- return EOF;
- fp = input->fp;
- if (!input->line)
- input->line = xmalloc(INPUTSIZ);
- bp = ptr = input->ptr = input->line;
-
- while ((c = getc(fp)) != EOF && isspace(c)) {
- if (c == '\n')
- newline();
- }
- if (c == EOF) {
- delinput();
- goto repeat;
- }
- ungetc(c, fp);
+ bp = input->line;
+ lim = bp + INPUTSIZ-1;
for (;;) {
c = readchar();
@@ -268,7 +215,7 @@ repeat:
}
if (c == '\n')
break;
- if (bp == &ptr[INPUTSIZ-1])
+ if (bp == lim)
die("line %d too big in file '%s'",
input->nline, input->fname);
if (c == '/') {
@@ -284,60 +231,88 @@ repeat:
}
*bp++ = c;
}
+ ungetc(c, input->fp);
+ *bp = '\0';
+}
+
+static bool
+fill(void)
+{
+ int c;
+ char *p;
+ FILE *fp;
- *bp = ' ';
- input->cnt = bp - ptr;
+repeat:
+ if (!input)
+ return 0;
+ if (input->begin && *input->begin != '\0')
+ return 1;
- if ((c = *input->ptr++) == '#') {
- *bp = '\0';
- preprocessor();
+ fp = input->fp;
+ if (!input->line)
+ input->line = xmalloc(INPUTSIZ);
+
+ while ((c = getc(fp)) != EOF && (c == '\n')) {
+ if (++input->nline == 0)
+ die("input file '%s' too long", input->fname);
+ }
+ if (c == EOF) {
+ delinput();
goto repeat;
}
- return c;
+ ungetc(c, fp);
+ readline();
+ if ((p = preprocessor(input->line)) == NULL)
+ goto repeat;
+ input->begin = input->p = p;
+ return 1;
}
-static int
-backchar(int c)
+static void
+tok2str(void)
{
- if (!input) {
- assert(c == EOF);
- return c;
- }
- ++input->cnt;
- return *--input->ptr = c;
+ size_t len;
+
+ if ((len = input->p - input->begin) > IDENTSIZ)
+ error("token too big");
+ strncpy(yytext, input->begin, len);
+ yytext[len] = '\0';
+ fprintf(stderr ,"%s\n", yytext);
+ input->begin = input->p;
}
static uint8_t
integer(char *s, char base)
{
- static Type *tp;
- static Symbol *sym;
- static char ch, size, sign;
- static long v;
-
- size = sign = 0;
-type:
- switch (ch = toupper(nextchar())) {
- case 'L':
- if (size == LLONG)
- goto wrong_type;
- size = (size == LONG) ? LLONG : LONG;
- goto type;
- case 'U':
- if (sign == UNSIGNED)
- goto wrong_type;
- goto type;
- default:
- backchar(ch);
- tp = ctype(INT, sign, size);
- break;
- wrong_type:
- error("invalid suffix in integer constant");
+ Type *tp;
+ Symbol *sym;
+ uint8_t size, sign;
+ long v;
+
+ for (size = sign = 0; ; ++input->p) {
+ switch (toupper(*input->p)) {
+ case 'L':
+ if (size == LLONG)
+ goto wrong_type;
+ size = (size == LONG) ? LLONG : LONG;
+ continue;
+ case 'U':
+ if (sign == UNSIGNED)
+ goto wrong_type;
+ sign = UNSIGNED;
+ continue;
+ default:
+ goto convert;
+ wrong_type:
+ error("invalid suffix in integer constant");
+ }
}
+convert:
+ tp = ctype(INT, sign, size);
sym = install("", NS_IDEN);
sym->type = tp;
- v = strtol(yytext, NULL, base);
+ v = strtol(s, NULL, base);
if (tp == inttype)
sym->u.i = v;
yylval.sym = sym;
@@ -347,30 +322,27 @@ type:
static char *
digits(uint8_t base)
{
- char ch, *bp;
+ char c, *p;
- for (bp = yytext ; bp < &yytext[IDENTSIZ]; *bp++ = ch) {
- ch = nextchar();
+ for (p = input->p; c = *p; ++p) {
switch (base) {
case 8:
- if (ch >= '7')
+ if (c > '7' || c < '0')
goto end;
- /* passthru */
+ break;
case 10:
- if (!isdigit(ch))
+ if (!isdigit(c))
goto end;
break;
case 16:
- if (!isxdigit(ch))
+ if (!isxdigit(c))
goto end;
break;
}
}
end:
- if (bp == &yytext[IDENTSIZ])
- error("number too long %s", yytext);
- *bp = '\0';
- backchar(ch);
+ input->p = p;
+ tok2str();
return yytext;
}
@@ -378,63 +350,51 @@ static uint8_t
number(void)
{
int ch;
- static char base;
+ char base;
- if ((ch = nextchar()) == '0') {
- if (toupper(ch = nextchar()) == 'X') {
+ if (*input->p != '0') {
+ base = 10;
+ } else {
+ if (toupper(*++input->p) == 'X') {
+ ++input->p;
base = 16;
} else {
base = 8;
- backchar(ch);
}
- } else {
- base = 10;
- backchar(ch);
}
return integer(digits(base), base);
}
-static char *
-escape(char *s)
-{
- uint8_t base;
- int c;
-
-repeat:
- switch (nextchar()) {
- case '\\': c = '\''; break;
- case 'a': c = '\a'; break;
- case 'f': c = '\f'; break;
- case 'n': c = '\n'; break;
- case 'r': c = '\r'; break;
- case 't': c = '\t'; break;
- case 'v': c = '\v'; break;
- case '\'': c = '\\'; break;
- case '"': c ='"'; break;
- case '?': c = '?'; break;
- case 'u': /* TODO: */
- case 'x':
- base = 16;
- goto number;
- case '0':
- base = 8;
- number:
- if ((c = atoi(digits(base))) > 255)
- warn("character constant out of range");
- break;
- case '\n':
- newline();
- if ((c = nextchar()) == '\\')
- goto repeat;
- break;
+static char
+escape(void)
+{
+ int c, base;
+
+ ++input->p;
+ switch (*input->p++) {
+ case '\\': return '\\';
+ case 'a': return '\a';
+ case 'f': return '\f';
+ case 'n': return '\n';
+ case 'r': return '\r';
+ case 't': return '\t';
+ case 'v': return '\v';
+ case '\'': return '\\';
+ case '"': return'"';
+ case '?': return '?';
+ case 'u': base = 10; break;
+ case 'x': base = 16; break;
+ case '0': base = 8; break;
default:
warn("unknown escape sequence");
- return s;
+ return ' ';
}
-
- *s = c;
- return ++s;
+ errno = 0;
+ c = strtoul(input->p, &input->p, base);
+ if (errno || c > 255)
+ warn("character constant out of range");
+ return c;
}
static uint8_t
@@ -443,12 +403,12 @@ character(void)
static char c;
Symbol *sym;
- nextchar(); /* discard the initial ' */
- c = nextchar();
- if (c == '\\')
- escape(&c);
- if (nextchar() != '\'')
+ if ((c = *++input->p) == '\\')
+ c = escape();
+ if (*input->p != '\'')
error("invalid character constant");
+ ++input->p;
+
sym = install("", NS_IDEN);
sym->u.i = c;
sym->type = inttype;
@@ -460,29 +420,20 @@ static uint8_t
string(void)
{
static char buf[STRINGSIZ+1];
- char *bp;
- int c;
- static Symbol *sym;
-
- nextchar(); /* discard the initial " */
-
- for (bp = buf; bp < &buf[STRINGSIZ]; ) {
- switch (c = nextchar()) {
- case EOF:
- error("found EOF while parsing");
- case '"':
- goto end_string;
- case '\\':
- bp = escape(bp);
+ Symbol *sym;
+ char *bp = buf, c;
+
+ assert(STRINGSIZ <= INPUTSIZ);
+ for (++input->p; (c = *input->p) != '\0'; ++input->p) {
+ if (c == '"')
break;
- default:
- *bp++ = c;
- }
+ if (c == '\\')
+ c = escape();
+ *bp++ = c;
}
-end_string:
- if (bp == &buf[IDENTSIZ])
- error("string too long");
+ if (c == '\0')
+ error("missing terminating '\"' character");
*bp = '\0';
sym = install("", NS_IDEN);
sym->u.s = xstrdup(buf);
@@ -494,19 +445,13 @@ end_string:
static uint8_t
iden(void)
{
- char *bp;
- int c;
+ char c, *p;
Symbol *sym;
- for (bp = yytext; bp < &yytext[IDENTSIZ]; *bp++ = c) {
- if (!isalnum(c = nextchar()) && c != '_')
- break;
- }
- if (bp == &yytext[IDENTSIZ])
- error("identifier too long %s", yytext);
- *bp = '\0';
- backchar(c);
-
+ for (p = input->p; isalpha(*p); ++p)
+ /* nothing */;
+ input->p = p;
+ tok2str();
sym = yylval.sym = lookup(yytext, lex_ns);
if (!sym || sym->token == IDEN)
return IDEN;
@@ -517,146 +462,106 @@ iden(void)
static uint8_t
follow(int expect, int ifyes, int ifno)
{
- int c = nextchar();
-
- if (c == expect) {
- yytext[1] = c;
- yytext[2] = 0;
+ if (*input->p++)
return ifyes;
- }
- backchar(c);
+ --input->p;
return ifno;
}
static uint8_t
minus(void)
{
- int c = nextchar();
-
- yytext[1] = c;
- yytext[2] = '\0';
- switch (c) {
+ switch (*input->p++) {
case '-': return DEC;
case '>': return INDIR;
case '=': return SUB_EQ;
- default:
- yytext[1] = '\0';
- backchar(c);
- return '-';
+ default: --input->p; return '-';
}
}
static uint8_t
plus(void)
{
- int c = nextchar();
-
- yytext[1] = c;
- yytext[2] = '\0';
- switch (c) {
+ switch (*input->p++) {
case '+': return INC;
case '=': return ADD_EQ;
- default:
- yytext[1] = '\0';
- backchar(c);
- return '+';
+ default: --input->p; return '+';
}
}
static uint8_t
relational(uint8_t op, uint8_t equal, uint8_t shift, uint8_t assig)
{
- int c = nextchar();
-
- yytext[1] = c;
- yytext[2] = '\0';
+ char c;
- if (c == '=')
+ if ((c = *input->p++) == '=')
return equal;
if (c == op)
return follow('=', assig, shift);
- backchar(c);
- yytext[1] = '\0';
+ --input->p;
return op;
}
static uint8_t
logic(uint8_t op, uint8_t equal, uint8_t logic)
{
- int c = nextchar();
+ char c;
- yytext[1] = c;
- yytext[2] = '\0';
-
- if (c == '=')
+ if ((c = *input->p++) == equal)
return equal;
if (c == op)
return logic;
- backchar(c);
- yytext[1] = '\0';
+ --input->p;
return op;
}
static uint8_t
dot(void)
{
- int c;
+ char c;
- if ((c = nextchar()) != '.') {
- backchar(c);
+ if (c = *input->p != '.')
return '.';
- } else if ((c = nextchar()) != '.') {
- error("incorrect token '%s'", yytext);
- } else {
- yytext[2] = yytext[1] = '.';
- yytext[3] = '\0';
- return ELLIPSIS;
- }
+ if ((c = *++input->p) != '.')
+ error("incorrect token '..'");
+ ++input->p;
+ return ELLIPSIS;
}
static uint8_t
operator(void)
{
- uint8_t c = nextchar();
-
- yytext[0] = c;
- yytext[1] = '\0';
- switch (c) {
- case '<': return relational('<', LE, SHL, SHL_EQ);
- case '>': return relational('>', GE, SHR, SHR_EQ);
- case '&': return logic('&', AND_EQ, AND);
- case '|': return logic('|', OR_EQ, OR);
- case '=': return follow('=', EQ, '=');
- case '^': return follow('=', XOR_EQ, '^');
- case '*': return follow('=', MUL_EQ, '*');
- case '/': return follow('=', DIV_EQ, '/');
- case '!': return follow('=', NE, '!');
- case '-': return minus();
- case '+': return plus();
- case '.': return dot();
- default: return c;
+ uint8_t t;
+
+ switch (t = *input->p++) {
+ case '<': t = relational('<', LE, SHL, SHL_EQ); break;
+ case '>': t = relational('>', GE, SHR, SHR_EQ); break;
+ case '&': t = logic('&', AND_EQ, AND); break;
+ case '|': t = logic('|', OR_EQ, OR); break;
+ case '=': t = follow('=', EQ, '='); break;
+ case '^': t = follow('=', XOR_EQ, '^'); break;
+ case '*': t = follow('=', MUL_EQ, '*'); break;
+ case '/': t = follow('=', DIV_EQ, '/'); break;
+ case '!': t = follow('=', NE, '!'); break;
+ case '-': t = minus(); break;
+ case '+': t = plus(); break;
+ case '.': t = dot(); break;
}
-}
-
-static int
-skipspaces(void)
-{
-
- int c;
-
- while (isspace(c = nextchar())) {
- if (c == '\n')
- newline();
- }
- return c;
+ tok2str();
+ return t;
}
uint8_t
next(void)
{
- int c;
+ char c;
- backchar(c = skipspaces());
+ if (!fill())
+ return EOFTOK;
+
+ while (isspace(*input->begin))
+ ++input->begin;
+ c = *(input->p = input->begin);
if (isalpha(c) || c == '_') {
yytoken = iden();
@@ -666,9 +571,6 @@ next(void)
yytoken = string();
} else if (c == '\'') {
yytoken = character();
- } else if (c == EOF) {
- strcpy(yytext, "EOF");
- yytoken = EOFTOK;
} else {
yytoken = operator();
}
@@ -693,8 +595,15 @@ ahead(void)
{
int c;
- backchar(c = skipspaces());
-
+repeat:
+ if (!input)
+ return EOFTOK;
+ while (isspace(c = *input->begin))
+ ;
+ if (c == '\0') {
+ fill();
+ goto repeat;
+ }
return c;
}
@@ -708,10 +617,9 @@ void
discard(void)
{
extern jmp_buf recover;
- int c;
+ char c;
- c = yytoken;
- do {
+ for (c = yytoken; ; c = *input->p++) {
switch (safe) {
case END_COMP:
if (c == '}')
@@ -730,7 +638,11 @@ discard(void)
goto jump;
break;
}
- } while ((c = nextchar()) != EOF);
+ if (*input->p == '\0')
+ fill();
+ if (!input)
+ break;
+ }
c = EOFTOK;
jump:
diff --git a/cc1/symbol.c b/cc1/symbol.c
@@ -107,13 +107,10 @@ install(char *s, uint8_t ns)
void
ikeywords(void)
{
- extern void include(char *), define(char *), undef(char *);
- extern void ifdef(char *), ifndef(char *), endif(char *);
- static struct words {
+ static struct {
char *str;
uint8_t token, value;
- void (*fun)(char *);
- } ccwords[] = {
+ } *bp, buff[] = {
{"auto", SCLASS, AUTO},
{"break", BREAK, BREAK},
{"_Bool", TYPE, BOOL},
@@ -149,36 +146,13 @@ ikeywords(void)
{"volatile", TQUALIFIER, VOLATILE},
{"while", WHILE, WHILE},
{NULL, 0, 0},
- }, cppwords[] = {
- {"include", 0, 0, include},
- {"define", 0, 0, define},
- {"undef", 0, 0, undef},
- {"ifdef", 0, 0, ifdef},
- {"ifndef", 0, 0, ifndef},
- {"endif", 0, 0, endif},
- {NULL, 0, 0}
};
- static struct wordlist {
- struct words *words;
- uint8_t ns;
- } wordlist [] = {
- {ccwords, NS_IDEN},
- {cppwords, NS_CPP},
- {NULL, 0}
- };
- struct wordlist *lp;
- struct words *bp;
Symbol *sym;
- for (lp = wordlist; lp->words; ++lp) {
- for (bp = lp->words; bp->str; ++bp) {
- sym = install(bp->str, lp->ns);
- sym->token = bp->token;
- if (bp->fun)
- sym->u.fun = bp->fun;
- else
- sym->u.token = bp->value;
- }
+ for (bp = buff; bp->str; ++bp) {
+ sym = install(bp->str, NS_IDEN);
+ sym->token = bp->token;
+ sym->u.token = bp->value;
}
globalcnt = 0;
}
