radare2/shlr/tcc/tccpp.c

/*
 *  TCC - Tiny C Compiler
 *
 *  Copyright (c) 2001-2004 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "tcc.h"
#include <math.h>
/********************************************************/
/* global variables */

ST_DATA int tok_flags;
/* additional informations about token */
#define TOK_FLAG_BOL   0x0001	/* beginning of line before */
#define TOK_FLAG_BOF   0x0002	/* beginning of file before */
#define TOK_FLAG_ENDIF 0x0004	/* a endif was found matching starting #ifdef */
#define TOK_FLAG_EOF   0x0008	/* end of file */

ST_DATA int parse_flags;
#define PARSE_FLAG_PREPROCESS 0x0001	/* activate preprocessing */
#define PARSE_FLAG_TOK_NUM    0x0002	/* return numbers instead of TOK_PPNUM */
#define PARSE_FLAG_LINEFEED   0x0004	/* line feed is returned as a
					token. line feed is also
					returned at eof */
#define PARSE_FLAG_ASM_COMMENTS 0x0008	/* '#' can be used for line comment */
#define PARSE_FLAG_SPACES     0x0010	/* next() returns space tokens (for -E) */

ST_DATA struct BufferedFile *file;
ST_DATA int ch, tok;
ST_DATA CValue tokc;
ST_DATA const int *macro_ptr;
ST_DATA CString tokcstr;/* current parsed string, if any */

/* display benchmark infos */
ST_DATA int total_lines;
ST_DATA int total_bytes;
ST_DATA int tok_ident;
ST_DATA TokenSym **table_ident;

/* ------------------------------------------------------------------------- */

static int *macro_ptr_allocated;
static const int *unget_saved_macro_ptr;
static int unget_saved_buffer[TOK_MAX_SIZE + 1];
static int unget_buffer_enabled;
static TokenSym *hash_ident[TOK_HASH_SIZE];
static char token_buf[STRING_MAX_SIZE + 1];
/* true if isid(c) || isnum(c) || isdot(c) */
static unsigned char isidnum_table[256 - CH_EOF];

static const char tcc_keywords[] =
#define DEF(id, str) str "\0"
#include "tcctok.h"
#undef DEF
;

/* WARNING: the content of this string encodes token numbers */
static const unsigned char tok_two_chars[] =
	"<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
	"-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";

struct macro_level {
	struct macro_level *prev;
	const int *p;
};

static void next_nomacro_spc(void);
static void macro_subst(
	TokenString *tok_str,
	Sym **nested_list,
	const int *macro_str,
	struct macro_level **can_read_stream
);

ST_FUNC void skip(int c)
{
	if (tok != c) {
		tcc_error ("'%c' expected (got \"%s\")", c, get_tok_str (tok, &tokc));
	}
	next ();
}

ST_FUNC void expect(const char *msg)
{
	tcc_error ("%s expected", msg);
}

/* ------------------------------------------------------------------------- */
/* CString handling */
static void cstr_realloc(CString *cstr, int new_size)
{
	int size;
	void *data;

	size = cstr->size_allocated;
	if (size == 0) {
		size = 8;	/* no need to allocate a too small first string */
	}
	while (size < new_size)
		size = size * 2;
	data = realloc (cstr->data_allocated, size);
	cstr->data_allocated = data;
	cstr->size_allocated = size;
	cstr->data = data;
}

/* add a byte */
ST_FUNC void cstr_ccat(CString *cstr, int ch)
{
	int size;
	size = cstr->size + 1;
	if (size > cstr->size_allocated) {
		cstr_realloc (cstr, size);
	}
	((unsigned char *) cstr->data)[size - 1] = ch;
	cstr->size = size;
}

ST_FUNC void cstr_cat(CString *cstr, const char *str)
{
	int c;
	for (;;) {
		c = *str;
		if (c == '\0') {
			break;
		}
		cstr_ccat (cstr, c);
		str++;
	}
}

/* add a wide char */
ST_FUNC void cstr_wccat(CString *cstr, int ch)
{
	int size;
	size = cstr->size + sizeof(nwchar_t);
	if (size > cstr->size_allocated) {
		cstr_realloc (cstr, size);
	}
	*(nwchar_t *) (((unsigned char *) cstr->data) + size - sizeof(nwchar_t)) = ch;
	cstr->size = size;
}

ST_FUNC void cstr_new(CString *cstr)
{
	memset (cstr, 0, sizeof(CString));
}

/* free string and reset it to NULL */
ST_FUNC void cstr_free(CString *cstr)
{
	free (cstr->data_allocated);
	cstr_new (cstr);
}

/* reset string to empty */
ST_FUNC void cstr_reset(CString *cstr)
{
	cstr->size = 0;
}

/* XXX: unicode ? */
static void add_char(CString *cstr, int c)
{
	if (c == '\'' || c == '\"' || c == '\\') {
		/* XXX: could be more precise if char or string */
		cstr_ccat (cstr, '\\');
	}
	if (c >= 32 && c <= 126) {
		cstr_ccat (cstr, c);
	} else {
		cstr_ccat (cstr, '\\');
		if (c == '\n') {
			cstr_ccat (cstr, 'n');
		} else {
			cstr_ccat (cstr, '0' + ((c >> 6) & 7));
			cstr_ccat (cstr, '0' + ((c >> 3) & 7));
			cstr_ccat (cstr, '0' + (c & 7));
		}
	}
}

/* ------------------------------------------------------------------------- */
/* allocate a new token */
static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
{
	TokenSym *ts, **ptable;
	int i;

	if (tok_ident >= SYM_FIRST_ANOM) {
		tcc_error ("memory full");
	}

	/* expand token table if needed */
	i = tok_ident - TOK_IDENT;
	if ((i % TOK_ALLOC_INCR) == 0) {
		ptable = realloc (table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
		table_ident = ptable;
	}
	ts = malloc (sizeof(TokenSym) + len);
	table_ident[i] = ts;
	ts->tok = tok_ident++;
	ts->sym_define = NULL;
	ts->sym_label = NULL;
	ts->sym_struct = NULL;
	ts->sym_identifier = NULL;
	ts->len = len;
	ts->hash_next = NULL;
	memcpy (ts->str, str, len);
	ts->str[len] = '\0';
	*pts = ts;
	return ts;
}

#define TOK_HASH_INIT 1
#define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))

/* find a token and add it if not found */
ST_FUNC TokenSym *tok_alloc(const char *str, int len)
{
	TokenSym *ts, **pts;
	int i;
	unsigned int h;

	h = TOK_HASH_INIT;
	for (i = 0; i < len; i++) {
		h = TOK_HASH_FUNC (h, ((unsigned char *) str)[i]);
	}
	h &= (TOK_HASH_SIZE - 1);

	pts = &hash_ident[h];
	for (;;) {
		ts = *pts;
		if (!ts) {
			break;
		}
		if (ts->len == len && !memcmp (ts->str, str, len)) {
			return ts;
		}
		pts = &(ts->hash_next);
	}
	return tok_alloc_new (pts, str, len);
}

/* XXX: buffer overflow */
/* XXX: float tokens */
ST_FUNC char *get_tok_str(int v, CValue *cv)
{
	static char buf[STRING_MAX_SIZE + 1];
	static CString cstr_buf;
	CString *cstr;
	char *p;
	int i, len;

	/* NOTE: to go faster, we give a fixed buffer for small strings */
	cstr_reset (&cstr_buf);
	cstr_buf.data = buf;
	cstr_buf.size_allocated = sizeof(buf);
	p = buf;

	switch (v) {
	case TOK_CINT:
	case TOK_CUINT:
		/* XXX: not quite exact, but only useful for testing */
		if (cv) {
			sprintf (p, "%u", cv->ui);
		}
		break;
	case TOK_CLLONG:
	case TOK_CULLONG:
		/* XXX: not quite exact, but only useful for testing  */
		if (cv) {
			sprintf (p, "%"PFMT64u, cv->ull);
		}
		break;
	case TOK_LCHAR:
		cstr_ccat (&cstr_buf, 'L');
	case TOK_CCHAR:
		cstr_ccat (&cstr_buf, '\'');
		if (cv) {
			add_char (&cstr_buf, cv->i);
		}
		cstr_ccat (&cstr_buf, '\'');
		cstr_ccat (&cstr_buf, '\0');
		break;
	case TOK_PPNUM:
		cstr = cv->cstr;
		len = cstr->size - 1;
		for (i = 0; i < len; i++) {
			add_char (&cstr_buf, ((unsigned char *) cstr->data)[i]);
		}
		cstr_ccat (&cstr_buf, '\0');
		break;
	case TOK_LSTR:
		cstr_ccat (&cstr_buf, 'L');
	case TOK_STR:
		if (cv) {
			cstr = cv->cstr;
			cstr_ccat (&cstr_buf, '\"');
			if (v == TOK_STR) {
				len = cstr->size - 1;
				for (i = 0; i < len; i++) {
					add_char (&cstr_buf, ((unsigned char *) cstr->data)[i]);
				}
			} else {
				len = (cstr->size / sizeof(nwchar_t)) - 1;
				for (i = 0; i < len; i++) {
					add_char (&cstr_buf, ((nwchar_t *) cstr->data)[i]);
				}
			}
			cstr_ccat (&cstr_buf, '\"');
			cstr_ccat (&cstr_buf, '\0');
		} else {
			eprintf ("cv = nil\n");
		}
		break;
	case TOK_LT:
		v = '<';
		goto addv;
	case TOK_GT:
		v = '>';
		goto addv;
	case TOK_DOTS:
		return strcpy (p, "...");
	case TOK_A_SHL:
		return strcpy (p, "<<=");
	case TOK_A_SAR:
		return strcpy (p, ">>=");
	default:
		if (v < TOK_IDENT) {
			/* search in two bytes table */
			const unsigned char *q = tok_two_chars;
			while (*q) {
				if (q[2] == v) {
					*p++ = q[0];
					*p++ = q[1];
					*p = '\0';
					return buf;
				}
				q += 3;
			}
addv:
			*p++ = v;
			*p = '\0';
		} else if (v < tok_ident) {
			return table_ident[v - TOK_IDENT]->str;
		} else if (v >= SYM_FIRST_ANOM) {
			/* special name for anonymous symbol */
			sprintf (p, "%u", v - SYM_FIRST_ANOM);
		} else {
			/* should never happen */
			return NULL;
		}
		break;
	}
	return cstr_buf.data;
}

/* fill input buffer and peek next char */
static int tcc_peekc_slow(BufferedFile *bf)
{
	int len;
	/* only tries to read if really end of buffer */
	if (bf->buf_ptr >= bf->buf_end) {
		if (bf->fd != -1) {
#if defined(PARSE_DEBUG)
			len = 8;
#else
			len = IO_BUF_SIZE;
#endif
			len = read (bf->fd, bf->buffer, len);
			if (len < 0) {
				len = 0;
			}
		} else {
			len = 0;
		}
		total_bytes += len;
		bf->buf_ptr = bf->buffer;
		bf->buf_end = bf->buffer + len;
		*bf->buf_end = CH_EOB;
	}
	if (bf->buf_ptr < bf->buf_end) {
		return bf->buf_ptr[0];
	} else {
		bf->buf_ptr = bf->buf_end;
		return CH_EOF;
	}
}

/* return the current character, handling end of block if necessary
   (but not stray) */
ST_FUNC int handle_eob(void)
{
	return tcc_peekc_slow (file);
}

/* read next char from current input file and handle end of input buffer */
ST_INLN void inp(void)
{
	ch = *(++(file->buf_ptr));
	/* end of buffer/file handling */
	if (ch == CH_EOB) {
		ch = handle_eob ();
	}
}

/* handle '\[\r]\n' */
static int handle_stray_noerror(void)
{
	while (ch == '\\') {
		inp ();
		if (ch == '\n') {
			file->line_num++;
			inp ();
		} else if (ch == '\r') {
			inp ();
			if (ch != '\n') {
				goto fail;
			}
			file->line_num++;
			inp ();
		} else {
fail:
			return 1;
		}
	}
	return 0;
}

static void handle_stray(void)
{
	if (handle_stray_noerror ()) {
		tcc_error ("stray '\\' in program");
	}
}

/* skip the stray and handle the \\n case. Output an error if
   incorrect char after the stray */
static int handle_stray1(uint8_t *p)
{
	int c;

	if (p >= file->buf_end) {
		file->buf_ptr = p;
		c = handle_eob ();
		p = file->buf_ptr;
		if (c == '\\') {
			goto parse_stray;
		}
	} else {
parse_stray:
		file->buf_ptr = p;
		ch = *p;
		handle_stray ();
		p = file->buf_ptr;
		c = *p;
	}
	return c;
}

/* handle just the EOB case, but not stray */
#define PEEKC_EOB(c, p)			\
	{				\
		p++;			\
		c = *p;			\
		if (c == '\\') {	\
			file->buf_ptr = p;\
			c = handle_eob ();\
			p = file->buf_ptr;\
		}			\
	}

/* handle the complicated stray case */
#define PEEKC(c, p)			\
	{				\
		p++;			\
		c = *p;			\
		if (c == '\\') {	\
			c = handle_stray1 (p);\
			p = file->buf_ptr;\
		}			\
	}

/* input with '\[\r]\n' handling. Note that this function cannot
   handle other characters after '\', so you cannot call it inside
   strings or comments */
ST_FUNC void minp(void)
{
	inp ();
	if (ch == '\\') {
		handle_stray ();
	}
}


/* single line C++ comments */
static uint8_t *parse_line_comment(uint8_t *p)
{
	int c;

	p++;
	for (;;) {
		c = *p;
redo:
		if (c == '\n' || c == CH_EOF) {
			break;
		} else if (c == '\\') {
			file->buf_ptr = p;
			c = handle_eob ();
			p = file->buf_ptr;
			if (c == '\\') {
				PEEKC_EOB (c, p);
				if (c == '\n') {
					file->line_num++;
					PEEKC_EOB (c, p);
				} else if (c == '\r') {
					PEEKC_EOB (c, p);
					if (c == '\n') {
						file->line_num++;
						PEEKC_EOB (c, p);
					}
				}
			} else {
				goto redo;
			}
		} else {
			p++;
		}
	}
	return p;
}

/* C comments */
ST_FUNC uint8_t *parse_comment(uint8_t *p)
{
	int c;

	p++;
	for (;;) {
		/* fast skip loop */
		for (;;) {
			c = *p;
			if (c == '\n' || c == '*' || c == '\\') {
				break;
			}
			p++;
			c = *p;
			if (c == '\n' || c == '*' || c == '\\') {
				break;
			}
			p++;
		}
		/* now we can handle all the cases */
		if (c == '\n') {
			file->line_num++;
			p++;
		} else if (c == '*') {
			p++;
			for (;;) {
				c = *p;
				if (c == '*') {
					p++;
				} else if (c == '/') {
					goto end_of_comment;
				} else if (c == '\\') {
					file->buf_ptr = p;
					c = handle_eob ();
					p = file->buf_ptr;
					if (c == '\\') {
						/* skip '\[\r]\n', otherwise just skip the stray */
						while (c == '\\') {
							PEEKC_EOB (c, p);
							if (c == '\n') {
								file->line_num++;
								PEEKC_EOB (c, p);
							} else if (c == '\r') {
								PEEKC_EOB (c, p);
								if (c == '\n') {
									file->line_num++;
									PEEKC_EOB (c, p);
								}
							} else {
								goto after_star;
							}
						}
					}
				} else {
					break;
				}
			}
after_star:
			;
		} else {
			/* stray, eob or eof */
			file->buf_ptr = p;
			c = handle_eob ();
			p = file->buf_ptr;
			if (c == CH_EOF) {
				tcc_error ("unexpected end of file in comment");
			} else if (c == '\\') {
				p++;
			}
		}
	}
end_of_comment:
	p++;
	return p;
}

#define cinp minp

static inline void skip_spaces(void)
{
	while (is_space (ch))
		cinp ();
}

static inline int check_space(int t, int *spc)
{
	if (is_space (t)) {
		if (*spc) {
			return 1;
		}
		*spc = 1;
	} else {
		*spc = 0;
	}
	return 0;
}

/* parse a string without interpreting escapes */
static uint8_t *parse_pp_string(uint8_t *p,
				int sep, CString *str)
{
	int c;
	p++;
	while (tcc_nerr () == 0) {
		c = *p;
		if (c == sep) {
			break;
		} else if (c == '\\') {
			file->buf_ptr = p;
			c = handle_eob ();
			p = file->buf_ptr;
			if (c == CH_EOF) {
unterminated_string:
				/* XXX: indicate line number of start of string */
				tcc_error ("missing terminating %c character", sep);
				return NULL;
			} else if (c == '\\') {
				/* escape : just skip \[\r]\n */
				PEEKC_EOB (c, p);
				if (c == '\n') {
					file->line_num++;
					p++;
				} else if (c == '\r') {
					PEEKC_EOB (c, p);
					if (c != '\n') {
						expect ("'\n' after '\r'");
						return NULL;
					}
					file->line_num++;
					p++;
				} else if (c == CH_EOF) {
					goto unterminated_string;
				} else {
					if (str) {
						cstr_ccat (str, '\\');
						cstr_ccat (str, c);
					}
					p++;
				}
			}
		} else if (c == '\n') {
			file->line_num++;
			goto add_char;
		} else if (c == '\r') {
			PEEKC_EOB (c, p);
			if (c != '\n') {
				if (str) {
					cstr_ccat (str, '\r');
				}
			} else {
				file->line_num++;
				goto add_char;
			}
		} else {
add_char:
			if (str) {
				cstr_ccat (str, c);
			}
			p++;
		}
	}
	p++;
	return p;
}

/* skip block of text until #else, #elif or #endif. skip also pairs of
   #if/#endif */
static void preprocess_skip(void)
{
	int a, start_of_line, c, in_warn_or_error;
	uint8_t *p;

	p = file->buf_ptr;
	a = 0;
redo_start:
	start_of_line = 1;
	in_warn_or_error = 0;
	while (tcc_nerr () == 0) {
redo_no_start:
		c = *p;
		switch (c) {
		case ' ':
		case '\t':
		case '\f':
		case '\v':
		case '\r':
			p++;
			goto redo_no_start;
		case '\n':
			file->line_num++;
			p++;
			goto redo_start;
		case '\\':
			file->buf_ptr = p;
			c = handle_eob ();
			if (c == CH_EOF) {
				expect ("#endif");
				return;
			} else if (c == '\\') {
				ch = file->buf_ptr[0];
				handle_stray_noerror ();
			}
			p = file->buf_ptr;
			goto redo_no_start;
		/* skip strings */
		case '\"':
		case '\'':
			if (in_warn_or_error) {
				goto _default;
			}
			p = parse_pp_string (p, c, NULL);
			if (p == NULL) {
				return;
			}
			break;
		/* skip comments */
		case '/':
			if (in_warn_or_error) {
				goto _default;
			}
			file->buf_ptr = p;
			ch = *p;
			minp ();
			p = file->buf_ptr;
			if (ch == '*') {
				p = parse_comment (p);
			} else if (ch == '/') {
				p = parse_line_comment (p);
			}
			break;
		case '#':
			p++;
			if (start_of_line) {
				file->buf_ptr = p;
				next_nomacro ();
				p = file->buf_ptr;
				if (a == 0 &&
				    (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF)) {
					goto the_end;
				}
				if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF) {
					a++;
				} else if (tok == TOK_ENDIF) {
					a--;
				} else if (tok == TOK_ERROR || tok == TOK_WARNING) {
					in_warn_or_error = 1;
				} else if (tok == TOK_LINEFEED) {
					goto redo_start;
				}
			}
			break;
_default:
		default:
			p++;
			break;
		}
		start_of_line = 0;
	}
the_end:
	;
	file->buf_ptr = p;
}

/* ParseState handling */

/* XXX: currently, no include file info is stored. Thus, we cannot display
   accurate messages if the function or data definition spans multiple
   files */

/* save current parse state in 's' */
ST_FUNC void save_parse_state(ParseState *s)
{
	s->line_num = file->line_num;
	s->macro_ptr = macro_ptr;
	s->tok = tok;
	s->tokc = tokc;
}

/* restore parse state from 's' */
ST_FUNC void restore_parse_state(ParseState *s)
{
	file->line_num = s->line_num;
	macro_ptr = s->macro_ptr;
	tok = s->tok;
	tokc = s->tokc;
}

/* return the number of additional 'ints' necessary to store the
   token */
static inline int tok_ext_size(int t)
{
	switch (t) {
	/* 4 bytes */
	case TOK_CINT:
	case TOK_CUINT:
	case TOK_CCHAR:
	case TOK_LCHAR:
	case TOK_CFLOAT:
	case TOK_LINENUM:
		return 1;
	case TOK_STR:
	case TOK_LSTR:
	case TOK_PPNUM:
		tcc_error ("unsupported token");
		return 1;
	case TOK_CDOUBLE:
	case TOK_CLLONG:
	case TOK_CULLONG:
		return 2;
	case TOK_CLDOUBLE:
		return LDOUBLE_SIZE / 4;
	default:
		return 0;
	}
}

/* token string handling */

ST_INLN void tok_str_new(TokenString *s)
{
	s->str = NULL;
	s->len = 0;
	s->allocated_len = 0;
	s->last_line_num = -1;
}

ST_FUNC void tok_str_free(int *str)
{
	free (str);
}

static int *tok_str_realloc(TokenString *s)
{
	int *str, len;

	if (s->allocated_len == 0) {
		len = 8;
	} else {
		len = s->allocated_len * 2;
	}
	str = realloc (s->str, len * sizeof(int));
	s->allocated_len = len;
	s->str = str;
	return str;
}

ST_FUNC void tok_str_add(TokenString *s, int t)
{
	int len, *str;

	len = s->len;
	str = s->str;
	if (len >= s->allocated_len) {
		str = tok_str_realloc (s);
	}
	str[len++] = t;
	s->len = len;
}

static void tok_str_add2(TokenString *s, int t, CValue *cv)
{
	int len, *str;

	len = s->len;
	str = s->str;

	/* allocate space for worst case */
	if (len + TOK_MAX_SIZE > s->allocated_len) {
		str = tok_str_realloc (s);
	}
	str[len++] = t;
	switch (t) {
	case TOK_CINT:
	case TOK_CUINT:
	case TOK_CCHAR:
	case TOK_LCHAR:
	case TOK_CFLOAT:
	case TOK_LINENUM:
		str[len++] = cv->tab[0];
		break;
	case TOK_PPNUM:
	case TOK_STR:
	case TOK_LSTR:
	{
		int nb_words;

		nb_words = (sizeof(CString) + cv->cstr->size + 3) >> 2;
		while ((len + nb_words) > s->allocated_len) {
			str = tok_str_realloc (s);
		}
		CString cstr = {0};
		cstr.data = NULL;
		cstr.size = cv->cstr->size;
		cstr.data_allocated = NULL;
		cstr.size_allocated = cstr.size;

		ut8 *p = (ut8*)(str + len);
		memcpy (p, &cstr, sizeof (CString));
		memcpy (p + sizeof (CString),
			cv->cstr->data, cstr.size);
		len += nb_words;
	}
	break;
	case TOK_CDOUBLE:
	case TOK_CLLONG:
	case TOK_CULLONG:
#if LDOUBLE_SIZE == 8
	case TOK_CLDOUBLE:
#endif
		str[len++] = cv->tab[0];
		str[len++] = cv->tab[1];
		break;
#if LDOUBLE_SIZE == 12
	case TOK_CLDOUBLE:
		str[len++] = cv->tab[0];
		str[len++] = cv->tab[1];
		str[len++] = cv->tab[2];
#elif LDOUBLE_SIZE == 16
	case TOK_CLDOUBLE:
		str[len++] = cv->tab[0];
		str[len++] = cv->tab[1];
		str[len++] = cv->tab[2];
		str[len++] = cv->tab[3];
#elif LDOUBLE_SIZE != 8
#error add long double size support
#endif
		break;
	default:
		break;
	}
	s->len = len;
}

/* add the current parse token in token string 's' */
ST_FUNC void tok_str_add_tok(TokenString *s)
{
	CValue cval;

	/* save line number info */
	if (file->line_num != s->last_line_num) {
		s->last_line_num = file->line_num;
		cval.i = s->last_line_num;
		tok_str_add2 (s, TOK_LINENUM, &cval);
	}
	tok_str_add2 (s, tok, &tokc);
}

/* get a token from an integer array and increment pointer
   accordingly. we code it as a macro to avoid pointer aliasing. */
static inline void TOK_GET(int *t, const int **pp, CValue *cv)
{
	const int *p = *pp;
	int n, *tab;

	tab = cv->tab;
	switch (*t = *p++) {
	case TOK_CINT:
	case TOK_CUINT:
	case TOK_CCHAR:
	case TOK_LCHAR:
	case TOK_CFLOAT:
	case TOK_LINENUM:
		tab[0] = *p++;
		break;
	case TOK_STR:
	case TOK_LSTR:
	case TOK_PPNUM:
		cv->cstr = (CString *) p;
		cv->cstr->data = (char *) p + sizeof(CString);
		p += (sizeof(CString) + cv->cstr->size + 3) >> 2;
		break;
	case TOK_CDOUBLE:
	case TOK_CLLONG:
	case TOK_CULLONG:
		n = 2;
		goto copy;
	case TOK_CLDOUBLE:
#if LDOUBLE_SIZE == 16
		n = 4;
#elif LDOUBLE_SIZE == 12
		n = 3;
#elif LDOUBLE_SIZE == 8
		n = 2;
#else
#error add long double size support
#endif
copy:
		do {
			*tab++ = *p++;
		} while (--n);
		break;
	default:
		break;
	}
	*pp = p;
}

static int macro_is_equal(const int *a, const int *b)
{
	char buf[STRING_MAX_SIZE + 1];
	CValue cv;
	int t;
	while (*a && *b) {
		TOK_GET (&t, &a, &cv);
		pstrcpy (buf, sizeof buf, get_tok_str (t, &cv));
		TOK_GET (&t, &b, &cv);
		if (strcmp (buf, get_tok_str (t, &cv))) {
			return 0;
		}
	}
	return !(*a || *b);
}

/* defines handling */
ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
{
	Sym *s;

	s = define_find (v);
	if (s && !macro_is_equal (s->d, str)) {
		tcc_warning ("%s redefined", get_tok_str (v, NULL));
	}

	s = sym_push2 (&define_stack, v, macro_type, 0);
	if (!s) {
		return;
	}
	s->d = str;
	s->next = first_arg;
	if (v >= TOK_IDENT) {
		table_ident[v - TOK_IDENT]->sym_define = s;
	}
}

/* undefined a define symbol. Its name is just set to zero */
ST_FUNC void define_undef(Sym *s)
{
	int v;
	v = s->v;
	if (v >= TOK_IDENT && v < tok_ident) {
		table_ident[v - TOK_IDENT]->sym_define = NULL;
	}
	s->v = 0;
}

ST_INLN Sym *define_find(int v)
{
	v -= TOK_IDENT;
	if ((unsigned) v >= (unsigned) (tok_ident - TOK_IDENT)) {
		return NULL;
	}
	return table_ident[v]->sym_define;
}

/* free define stack until top reaches 'b' */
ST_FUNC void free_defines(Sym *b)
{
	Sym *top, *top1;
	int v;

	top = define_stack;
	while (top != b) {
		top1 = top->prev;
		/* do not free args or predefined defines */
		if (top->d) {
			tok_str_free (top->d);
		}
		v = top->v;
		if (v >= TOK_IDENT && v < tok_ident) {
			table_ident[v - TOK_IDENT]->sym_define = NULL;
		}
		sym_free (top);
		top = top1;
	}
	define_stack = b;
}

/* label lookup */
ST_FUNC Sym *label_find(int v)
{
	v -= TOK_IDENT;
	if ((unsigned) v >= (unsigned) (tok_ident - TOK_IDENT)) {
		return NULL;
	}
	return table_ident[v]->sym_label;
}

ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
{
	Sym *s, **ps;
	s = sym_push2 (ptop, v, 0, 0);
	if (!s) {
		return s;
	}
	s->r = flags;
	ps = &table_ident[v - TOK_IDENT]->sym_label;
	if (ptop == &global_label_stack) {
		/* modify the top most local identifier, so that
		   sym_identifier will point to 's' when popped */
		while (*ps != NULL)
			ps = &(*ps)->prev_tok;
	}
	s->prev_tok = *ps;
	*ps = s;
	return s;
}

/* pop labels until element last is reached. Look if any labels are
   undefined. Define symbols if '&&label' was used. */
ST_FUNC void label_pop(Sym **ptop, Sym *slast)
{
	Sym *s, *s1;
	for (s = *ptop; s != slast; s = s1) {
		s1 = s->prev;
		if (s->r == LABEL_DECLARED) {
			tcc_warning ("label '%s' declared but not used", get_tok_str (s->v, NULL));
		} else if (s->r == LABEL_FORWARD) {
			tcc_error ("label '%s' used but not defined",
				get_tok_str (s->v, NULL));
		}
		/* remove label */
		table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
		sym_free (s);
	}
	*ptop = slast;
}

/* eval an expression for #if/#elif */
static int expr_preprocess(void)
{
	int c, t;
	TokenString str;

	tok_str_new (&str);
	while (tok != TOK_LINEFEED && tok != TOK_EOF) {
		next ();/* do macro subst */
		if (tok == TOK_DEFINED) {
			next_nomacro ();
			t = tok;
			if (t == '(') {
				next_nomacro ();
			}
			c = define_find (tok) != 0;
			if (t == '(') {
				next_nomacro ();
			}
			tok = TOK_CINT;
			tokc.i = c;
		} else if (tok >= TOK_IDENT) {
			/* if undefined macro */
			tok = TOK_CINT;
			tokc.i = 0;
		}
		tok_str_add_tok (&str);
	}
	tok_str_add (&str, -1);	/* simulate end of file */
	tok_str_add (&str, 0);
	/* now evaluate C constant expression */
	macro_ptr = str.str;
	next ();
	c = expr_const ();
	macro_ptr = NULL;
	tok_str_free (str.str);
	return c != 0;
}

#if defined(PARSE_DEBUG) || defined(PP_DEBUG)
static void tok_print(int *str)
{
	int t;
	CValue cval;

	printf ("<");
	while (1) {
		TOK_GET (&t, &str, &cval);
		if (!t) {
			break;
		}
		printf ("%s", get_tok_str (t, &cval));
	}
	printf (">\n");
}
#endif

/* parse after #define */
ST_FUNC void parse_define(void)
{
	Sym *s, *first, **ps;
	int v, t, varg, is_vaargs, spc;
	TokenString str;

	v = tok;
	if (v < TOK_IDENT) {
		tcc_error ("invalid macro name '%s'", get_tok_str (tok, &tokc));
	}
	/* XXX: should check if same macro (ANSI) */
	first = NULL;
	t = MACRO_OBJ;
	/* '(' must be just after macro definition for MACRO_FUNC */
	next_nomacro_spc ();
	if (tok == '(') {
		next_nomacro ();
		ps = &first;
		while (tok != ')') {
			varg = tok;
			next_nomacro ();
			is_vaargs = 0;
			if (varg == TOK_DOTS) {
				varg = TOK___VA_ARGS__;
				is_vaargs = 1;
			} else if (tok == TOK_DOTS && gnu_ext) {
				is_vaargs = 1;
				next_nomacro ();
			}
			if (varg < TOK_IDENT) {
				tcc_error ("badly punctuated parameter list");
			}
			s = sym_push2 (&define_stack, varg | SYM_FIELD, is_vaargs, 0);
			if (!s) {
				return;
			}
			*ps = s;
			ps = &s->next;
			if (tok != ',') {
				break;
			}
			next_nomacro ();
		}
		if (tok == ')') {
			next_nomacro_spc ();
		}
		t = MACRO_FUNC;
	}
	tok_str_new (&str);
	spc = 2;
	/* EOF testing necessary for '-D' handling */
	while (tok != TOK_LINEFEED && tok != TOK_EOF) {
		/* remove spaces around ## and after '#' */
		if (TOK_TWOSHARPS == tok) {
			if (1 == spc) {
				--str.len;
			}
			spc = 2;
		} else if ('#' == tok) {
			spc = 2;
		} else if (check_space (tok, &spc)) {
			goto skip;
		}
		tok_str_add2 (&str, tok, &tokc);
skip:
		next_nomacro_spc ();
	}
	if (spc == 1) {
		--str.len;	/* remove trailing space */
	}
	tok_str_add (&str, 0);
#ifdef PP_DEBUG
	printf ("define %s %d: ", get_tok_str (v, NULL), t);
	tok_print (str.str);
#endif
	define_push (v, t, str.str, first);
}

static inline int hash_cached_include(const char *filename)
{
	const unsigned char *s;
	unsigned int h;

	h = TOK_HASH_INIT;
	s = (const unsigned char *) filename;
	while (*s) {
		h = TOK_HASH_FUNC (h, *s);
		s++;
	}
	h &= (CACHED_INCLUDES_HASH_SIZE - 1);
	return h;
}

static CachedInclude *search_cached_include(TCCState *s1, const char *filename)
{
	CachedInclude *e;
	int i, h;
	h = hash_cached_include (filename);
	i = s1->cached_includes_hash[h];
	for (;;) {
		if (i == 0) {
			break;
		}
		e = s1->cached_includes[i - 1];
		if (0 == PATHCMP (e->filename, filename)) {
			return e;
		}
		i = e->hash_next;
	}
	return NULL;
}

static inline void add_cached_include(TCCState *s1, const char *filename, int ifndef_macro)
{
	CachedInclude *e;
	int h;

	if (search_cached_include (s1, filename)) {
		return;
	}
#ifdef INC_DEBUG
	printf ("adding cached '%s' %s\n", filename, get_tok_str (ifndef_macro, NULL));
#endif
	e = malloc (sizeof(CachedInclude) + strlen (filename));
	strcpy (e->filename, filename);
	e->ifndef_macro = ifndef_macro;
	dynarray_add ((void ***) &s1->cached_includes, &s1->nb_cached_includes, e);
	/* add in hash table */
	h = hash_cached_include (filename);
	e->hash_next = s1->cached_includes_hash[h];
	s1->cached_includes_hash[h] = s1->nb_cached_includes;
}

static void pragma_parse(TCCState *s1)
{
	int val;

	next ();
	if (tok == TOK_pack) {
		/*
		  This may be:
		  #pragma pack(1) // set
		  #pragma pack() // reset to default
		  #pragma pack(push,1) // push & set
		  #pragma pack(pop) // restore previous
		*/
		next ();
		skip ('(');
		if (tok == TOK_ASM_pop) {
			next ();
			if (s1->pack_stack_ptr <= s1->pack_stack) {
stk_error:
				tcc_error ("out of pack stack");
			}
			s1->pack_stack_ptr--;
		} else {
			val = 0;
			if (tok != ')') {
				if (tok == TOK_ASM_push) {
					next ();
					if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1) {
						goto stk_error;
					}
					s1->pack_stack_ptr++;
					skip (',');
				}
				if (tok != TOK_CINT) {
pack_error:
					tcc_error ("invalid pack pragma");
				}
				val = tokc.i;
				if (val < 1 || val > 16 || (val & (val - 1)) != 0) {
					goto pack_error;
				}
				next ();
			}
			*s1->pack_stack_ptr = val;
			skip (')');
		}
	}
}

/* is_bof is true if first non space token at beginning of file */
ST_FUNC void preprocess(int is_bof)
{
	TCCState *s1 = tcc_state;
	int i, c, n, saved_parse_flags;
	char buf[1024], *q;
	Sym *s;

	saved_parse_flags = parse_flags;
	parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM |
		      PARSE_FLAG_LINEFEED;
	next_nomacro ();
redo:
	switch (tok) {
	case TOK_DEFINE:
		next_nomacro ();
		parse_define ();
		break;
	case TOK_UNDEF:
		next_nomacro ();
		s = define_find (tok);
		/* undefine symbol by putting an invalid name */
		if (s) {
			define_undef (s);
		}
		break;
	case TOK_INCLUDE:
	case TOK_INCLUDE_NEXT:
		ch = file->buf_ptr[0];
		/* XXX: incorrect if comments : use next_nomacro with a special mode */
		skip_spaces ();
		if (ch == '<') {
			c = '>';
			goto read_name;
		} else if (ch == '\"') {
			c = ch;
read_name:
			inp ();
			q = buf;
			while (ch != c && ch != '\n' && ch != CH_EOF) {
				if ((q - buf) < sizeof(buf) - 1) {
					*q++ = ch;
				}
				if (ch == '\\') {
					if (handle_stray_noerror () == 0) {
						--q;
					}
				} else {
					inp ();
				}
			}
			*q = '\0';
			minp ();
#if 0
			/* eat all spaces and comments after include */
			/* XXX: slightly incorrect */
			while (ch1 != '\n' && ch1 != CH_EOF)
				inp ();
#endif
		} else {
			/* computed #include : either we have only strings or
			   we have anything enclosed in '<>' */
			next ();
			buf[0] = '\0';
			if (tok == TOK_STR) {
				while (tok != TOK_LINEFEED) {
					if (tok != TOK_STR) {
include_syntax:
						tcc_error ("'#include' expects \"FILENAME\" or <FILENAME>");
					}
					pstrcat (buf, sizeof(buf), (char *) tokc.cstr->data);
					next ();
				}
				c = '\"';
			} else {
				int len;
				while (tok != TOK_LINEFEED) {
					pstrcat (buf, sizeof(buf), get_tok_str (tok, &tokc));
					next ();
				}
				len = strlen (buf);
				/* check syntax and remove '<>' */
				if (len < 2 || buf[0] != '<' || buf[len - 1] != '>') {
					goto include_syntax;
				}
				memmove (buf, buf + 1, len - 2);
				buf[len - 2] = '\0';
				c = '>';
			}
		}

		if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE) {
			tcc_error ("#include recursion too deep");
		}
		/* store current file in stack, but increment stack later below */
		*s1->include_stack_ptr = file;

		n = s1->nb_include_paths + s1->nb_sysinclude_paths;
		for (i = -2; i < n; ++i) {
			char buf1[sizeof file->filename];
			CachedInclude *e;
			BufferedFile **f;
			const char *path;

			if (i == -2) {
				/* check absolute include path */
				if (!IS_ABSPATH (buf)) {
					continue;
				}
				buf1[0] = 0;
				i = n;	/* force end loop */

			} else if (i == -1) {
				/* search in current dir if "header.h" */
				if (c != '\"') {
					continue;
				}
				path = file->filename;
				pstrncpy (buf1, path, tcc_basename (path) - path);

			} else {
				/* search in all the include paths */
				if (i < s1->nb_include_paths) {
					path = s1->include_paths[i];
				} else {
					path = s1->sysinclude_paths[i - s1->nb_include_paths];
				}
				pstrcpy (buf1, sizeof(buf1), path);
				pstrcat (buf1, sizeof(buf1), "/");
			}

			pstrcat (buf1, sizeof(buf1), buf);

			if (tok == TOK_INCLUDE_NEXT) {
				for (f = s1->include_stack_ptr; f >= s1->include_stack; --f) {
					if (0 == PATHCMP ((*f)->filename, buf1)) {
#ifdef INC_DEBUG
						printf ("%s: #include_next skipping %s\n", file->filename, buf1);
#endif
						goto include_trynext;
					}
				}
			}

			e = search_cached_include (s1, buf1);
			if (e && define_find (e->ifndef_macro)) {
				/* no need to parse the include because the 'ifndef macro'
				   is defined */
#ifdef INC_DEBUG
				printf ("%s: skipping cached %s\n", file->filename, buf1);
#endif
				goto include_done;
			}

			if (tcc_open (s1, buf1) < 0) {
include_trynext:
				continue;
			}
			fprintf (stderr, "#include \"%s\"\n", buf1);

#ifdef INC_DEBUG
			fprintf (stderr, "%s: including %s\n", file->prev->filename, file->filename);
#endif
			/* update target deps */
			dynarray_add ((void ***) &s1->target_deps, &s1->nb_target_deps,
				strdup (buf1));
			/* push current file in stack */
			++s1->include_stack_ptr;
			tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
			ch = file->buf_ptr[0];
			goto the_end;
		}
		/* load include file from the same directory as the parent */
		{
			char filepath[1024];
			int filepath_len;
			char *e = file->filename + strlen (file->filename);
			while (e > file->filename) {
				if (*e == R_SYS_DIR[0]) {
					break;
				}
				e--;
			}
			filepath_len = R_MIN ((size_t) (e - file->filename) + 1, sizeof (filepath) - 1);
			memcpy (filepath, file->filename, filepath_len);
			strcpy (filepath + filepath_len, buf);
			if (tcc_open (s1, filepath) < 0) {
				if (!dirname) {
					dirname = "/usr/include";
				}
				int len = snprintf (filepath, sizeof (filepath), "%s/%s", dirname, buf);
				if (len >= sizeof (filepath) || tcc_open (s1, filepath) < 0) {
					tcc_error ("include file '%s' not found", filepath);
				} else {
					fprintf (stderr, "#include \"%s\"\n", filepath);
					++s1->include_stack_ptr;
					tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
					ch = file->buf_ptr[0];
					goto the_end;
				}
			} else {
				fprintf (stderr, "#include \"%s\"\n", filepath);
				++s1->include_stack_ptr;
				tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
				ch = file->buf_ptr[0];
				goto the_end;
			}
		}
include_done:
		break;
	case TOK_IFNDEF:
		c = 1;
		goto do_ifdef;
	case TOK_IF:
		c = expr_preprocess ();
		goto do_if;
	case TOK_IFDEF:
		c = 0;
do_ifdef:
		next_nomacro ();
		if (tok < TOK_IDENT) {
			tcc_error ("invalid argument for '#if%sdef'", c? "n": "");
		}
		if (is_bof) {
			if (c) {
#ifdef INC_DEBUG
				printf ("#ifndef %s\n", get_tok_str (tok, NULL));
#endif
				file->ifndef_macro = tok;
			}
		}
		c = (define_find (tok) != 0) ^ c;
do_if:
		if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE) {
			tcc_error ("memory full");
		}
		*s1->ifdef_stack_ptr++ = c;
		goto test_skip;
	case TOK_ELSE:
		if (s1->ifdef_stack_ptr == s1->ifdef_stack) {
			tcc_error ("#else without matching #if");
		}
		if (s1->ifdef_stack_ptr[-1] & 2) {
			tcc_error ("#else after #else");
		}
		c = (s1->ifdef_stack_ptr[-1] ^= 3);
		goto test_else;
	case TOK_ELIF:
		if (s1->ifdef_stack_ptr == s1->ifdef_stack) {
			tcc_error ("#elif without matching #if");
		}
		c = s1->ifdef_stack_ptr[-1];
		if (c > 1) {
			tcc_error ("#elif after #else");
		}
		/* last #if/#elif expression was true: we skip */
		if (c == 1) {
			goto skip;
		}
		c = expr_preprocess ();
		s1->ifdef_stack_ptr[-1] = c;
test_else:
		if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1) {
			file->ifndef_macro = 0;
		}
test_skip:
		if (!(c & 1)) {
skip:
			preprocess_skip ();
			is_bof = 0;
			goto redo;
		}
		break;
	case TOK_ENDIF:
		if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr) {
			tcc_error ("#endif without matching #if");
		}
		s1->ifdef_stack_ptr--;
		/* '#ifndef macro' was at the start of file. Now we check if
		   an '#endif' is exactly at the end of file */
		if (file->ifndef_macro &&
		    s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
			file->ifndef_macro_saved = file->ifndef_macro;
			/* need to set to zero to avoid false matches if another
			   #ifndef at middle of file */
			file->ifndef_macro = 0;
			while (tok != TOK_LINEFEED)
				next_nomacro ();
			tok_flags |= TOK_FLAG_ENDIF;
			goto the_end;
		}
		break;
	case TOK_LINE:
		next ();
		if (tok != TOK_CINT) {
			tcc_error ("#line");
		}
		file->line_num = tokc.i - 1;	/* the line number will be incremented after */
		next ();
		if (tok != TOK_LINEFEED) {
			if (tok != TOK_STR) {
				tcc_error ("#line");
			}
			pstrcpy (file->filename, sizeof(file->filename),
				(char *) tokc.cstr->data);
		}
		break;
	case TOK_ERROR:
	case TOK_WARNING:
		c = tok;
		ch = file->buf_ptr[0];
		skip_spaces ();
		q = buf;
		while (ch != '\n' && ch != CH_EOF) {
			if ((q - buf) < sizeof(buf) - 1) {
				*q++ = ch;
			}
			if (ch == '\\') {
				if (handle_stray_noerror () == 0) {
					--q;
				}
			} else {
				inp ();
			}
		}
		*q = '\0';
		tcc_warning ("#%s %s", c == TOK_ERROR? "error": "warning", buf);
		break;
	case TOK_PRAGMA:
		pragma_parse (s1);
		break;
	default:
		if (tok == TOK_LINEFEED || tok == '!' || tok == TOK_PPNUM) {
			/* '!' is ignored to allow C scripts. numbers are ignored
			   to emulate cpp behaviour */
		} else {
			if (!(saved_parse_flags & PARSE_FLAG_ASM_COMMENTS)) {
				tcc_warning ("Ignoring unknown preprocessing directive #%s", get_tok_str (tok, &tokc));
			} else {
				/* this is a gas line comment in an 'S' file. */
				file->buf_ptr = parse_line_comment (file->buf_ptr);
				goto the_end;
			}
		}
		break;
	}
	/* ignore other preprocess commands or #! for C scripts */
	while (tok != TOK_LINEFEED)
		next_nomacro ();
the_end:
	parse_flags = saved_parse_flags;
}

/* evaluate escape codes in a string. */
static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
{
	int c, n;
	const uint8_t *p;

	p = buf;
	for (;;) {
		c = *p;
		if (c == '\0') {
			break;
		}
		if (c == '\\') {
			p++;
			/* escape */
			c = *p;
			switch (c) {
			case '0': case '1': case '2': case '3':
			case '4': case '5': case '6': case '7':
				/* at most three octal digits */
				n = c - '0';
				p++;
				c = *p;
				if (isoct (c)) {
					n = n * 8 + c - '0';
					p++;
					c = *p;
					if (isoct (c)) {
						n = n * 8 + c - '0';
						p++;
					}
				}
				c = n;
				goto add_char_nonext;
			case 'x':
			case 'u':
			case 'U':
				p++;
				n = 0;
				for (;;) {
					c = *p;
					if (c >= 'a' && c <= 'f') {
						c = c - 'a' + 10;
					} else if (c >= 'A' && c <= 'F') {
						c = c - 'A' + 10;
					} else if (isnum (c)) {
						c = c - '0';
					} else {
						break;
					}
					n = n * 16 + c;
					p++;
				}
				c = n;
				goto add_char_nonext;
			case 'a':
				c = '\a';
				break;
			case 'b':
				c = '\b';
				break;
			case 'f':
				c = '\f';
				break;
			case 'n':
				c = '\n';
				break;
			case 'r':
				c = '\r';
				break;
			case 't':
				c = '\t';
				break;
			case 'v':
				c = '\v';
				break;
			case 'e':
				if (!gnu_ext) {
					goto invalid_escape;
				}
				c = 27;
				break;
			case '\'':
			case '\"':
			case '\\':
			case '?':
				break;
			default:
invalid_escape:
				if (c >= '!' && c <= '~') {
					tcc_warning ("unknown escape sequence: \'\\%c\'", c);
				} else {
					tcc_warning ("unknown escape sequence: \'\\x%x\'", c);
				}
				break;
			}
		}
		p++;
add_char_nonext:
		if (!is_long) {
			cstr_ccat (outstr, c);
		} else {
			cstr_wccat (outstr, c);
		}
	}
	/* add a trailing '\0' */
	if (!is_long) {
		cstr_ccat (outstr, '\0');
	} else {
		cstr_wccat (outstr, '\0');
	}
}

/* we use 64 bit numbers */
#define BN_SIZE 2

/* bn = (bn << shift) | or_val */
static void bn_lshift(unsigned int *bn, int shift, int or_val)
{
	int i;
	unsigned int v;
	for (i = 0; i < BN_SIZE; i++) {
		v = bn[i];
		bn[i] = (v << shift) | or_val;
		or_val = v >> (32 - shift);
	}
}

static void bn_zero(unsigned int *bn)
{
	int i;
	for (i = 0; i < BN_SIZE; i++) {
		bn[i] = 0;
	}
}

/* parse number in null terminated string 'p' and return it in the
   current token */
static void parse_number(const char *p)
{
	int b, t, shift, frac_bits, s, exp_val, ch;
	char *q;
	unsigned int bn[BN_SIZE];
	double d;

	/* number */
	q = token_buf;
	ch = *p++;
	t = ch;
	ch = *p++;
	*q++ = t;
	b = 10;
	if (t == '.') {
		goto float_frac_parse;
	} else if (t == '0') {
		if (ch == 'x' || ch == 'X') {
			q--;
			ch = *p++;
			b = 16;
		} else if (tcc_ext && (ch == 'b' || ch == 'B')) {
			q--;
			ch = *p++;
			b = 2;
		}
	}
	/* parse all digits. cannot check octal numbers at this stage
	   because of floating point constants */
	while (1) {
		if (ch >= 'a' && ch <= 'f') {
			t = ch - 'a' + 10;
		} else if (ch >= 'A' && ch <= 'F') {
			t = ch - 'A' + 10;
		} else if (isnum (ch)) {
			t = ch - '0';
		} else {
			break;
		}
		if (t >= b) {
			break;
		}
		if (q >= token_buf + STRING_MAX_SIZE) {
num_too_long:
			tcc_error ("number too long");
		}
		*q++ = ch;
		ch = *p++;
	}
	if (ch == '.' ||
	    ((ch == 'e' || ch == 'E') && b == 10) ||
	    ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
		if (b != 10) {
			/* NOTE: strtox should support that for hexa numbers, but
			   non ISOC99 libcs do not support it, so we prefer to do
			   it by hand */
			/* hexadecimal or binary floats */
			/* XXX: handle overflows */
			*q = '\0';
			if (b == 16) {
				shift = 4;
			} else {
				shift = 2;
			}
			bn_zero (bn);
			q = token_buf;
			while (1) {
				t = *q++;
				if (t == '\0') {
					break;
				} else if (t >= 'a') {
					t = t - 'a' + 10;
				} else if (t >= 'A') {
					t = t - 'A' + 10;
				} else {
					t = t - '0';
				}
				bn_lshift (bn, shift, t);
			}
			frac_bits = 0;
			if (ch == '.') {
				ch = *p++;
				while (1) {
					t = ch;
					if (t >= 'a' && t <= 'f') {
						t = t - 'a' + 10;
					} else if (t >= 'A' && t <= 'F') {
						t = t - 'A' + 10;
					} else if (t >= '0' && t <= '9') {
						t = t - '0';
					} else {
						break;
					}
					if (t >= b) {
						tcc_error ("invalid digit");
					}
					bn_lshift (bn, shift, t);
					frac_bits += shift;
					ch = *p++;
				}
			}
			if (ch != 'p' && ch != 'P') {
				expect ("exponent");
				return;
			}
			ch = *p++;
			s = 1;
			exp_val = 0;
			if (ch == '+') {
				ch = *p++;
			} else if (ch == '-') {
				s = -1;
				ch = *p++;
			}
			if (ch < '0' || ch > '9') {
				expect ("exponent digits");
				return;
			}
			while (ch >= '0' && ch <= '9') {
				exp_val = exp_val * 10 + ch - '0';
				ch = *p++;
			}
			exp_val = exp_val * s;

			/* now we can generate the number */
			/* XXX: should patch directly float number */
			d = (double) bn[1] * 4294967296.0 + (double) bn[0];
			d = ldexp (d, exp_val - frac_bits);
			t = toup (ch);
			if (t == 'F') {
				ch = *p++;
				tok = TOK_CFLOAT;
				/* float : should handle overflow */
				tokc.f = (float) d;
			} else if (t == 'L') {
				ch = *p++;
#ifdef TCC_TARGET_PE
				tok = TOK_CDOUBLE;
				tokc.d = d;
#else
				tok = TOK_CLDOUBLE;
				/* XXX: not large enough */
				tokc.ld = (long double) d;
#endif
			} else {
				tok = TOK_CDOUBLE;
				tokc.d = d;
			}
		} else {
			/* decimal floats */
			if (ch == '.') {
				if (q >= token_buf + STRING_MAX_SIZE) {
					goto num_too_long;
				}
				*q++ = ch;
				ch = *p++;
float_frac_parse:
				while (ch >= '0' && ch <= '9') {
					if (q >= token_buf + STRING_MAX_SIZE) {
						goto num_too_long;
					}
					*q++ = ch;
					ch = *p++;
				}
			}
			if (ch == 'e' || ch == 'E') {
				if (q >= token_buf + STRING_MAX_SIZE) {
					goto num_too_long;
				}
				*q++ = ch;
				ch = *p++;
				if (ch == '-' || ch == '+') {
					if (q >= token_buf + STRING_MAX_SIZE) {
						goto num_too_long;
					}
					*q++ = ch;
					ch = *p++;
				}
				if (ch < '0' || ch > '9') {
					expect ("exponent digits");
					return;
				}
				while (ch >= '0' && ch <= '9') {
					if (q >= token_buf + STRING_MAX_SIZE) {
						goto num_too_long;
					}
					*q++ = ch;
					ch = *p++;
				}
			}
			*q = '\0';
			t = toup (ch);
			errno = 0;
			if (t == 'F') {
				ch = *p++;
				tok = TOK_CFLOAT;
				tokc.f = strtof (token_buf, NULL);
			} else if (t == 'L') {
				ch = *p++;
				tok = TOK_CDOUBLE;
				tokc.d = strtod (token_buf, NULL);
#if 0
#ifdef TCC_TARGET_PE
				tok = TOK_CDOUBLE;
				tokc.d = strtod (token_buf, NULL);
#else
				tok = TOK_CLDOUBLE;
				tokc.ld = strtold (token_buf, NULL);
#endif
#endif
			} else {
				tok = TOK_CDOUBLE;
				tokc.d = strtod (token_buf, NULL);
			}
		}
	} else {
		unsigned long long n, n1;
		int lcount, ucount;

		/* integer number */
		*q = '\0';
		q = token_buf;
		if (b == 10 && *q == '0') {
			b = 8;
			q++;
		}
		n = 0;
		while (1) {
			t = *q++;
			/* no need for checks except for base 10 / 8 errors */
			if (t == '\0') {
				break;
			} else if (t >= 'a') {
				t = t - 'a' + 10;
			} else if (t >= 'A') {
				t = t - 'A' + 10;
			} else {
				t = t - '0';
				if (t >= b) {
					tcc_error ("invalid digit");
				}
			}
			n1 = n;
			n = n * b + t;
			/* detect overflow */
			/* XXX: this test is not reliable */
			if (n < n1) {
				tcc_error ("integer constant overflow");
			}
		}

		/* XXX: not exactly ANSI compliant */
		if ((n & 0xffffffff00000000LL) != 0) {
			if ((n >> 63) != 0) {
				tok = TOK_CULLONG;
			} else {
				tok = TOK_CLLONG;
			}
		} else if (n > 0x7fffffff) {
			tok = TOK_CUINT;
		} else {
			tok = TOK_CINT;
		}
		lcount = 0;
		ucount = 0;
		for (;;) {
			t = toup (ch);
			if (t == 'L') {
				if (lcount >= 2) {
					tcc_error ("three 'l's in integer constant");
				}
				lcount++;
#if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
				if (lcount == 2) {
#endif
				if (tok == TOK_CINT) {
					tok = TOK_CLLONG;
				} else if (tok == TOK_CUINT) {
					tok = TOK_CULLONG;
				}
#if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
			}
#endif
				ch = *p++;
			} else if (t == 'U') {
				if (ucount >= 1) {
					tcc_error ("two 'u's in integer constant");
				}
				ucount++;
				if (tok == TOK_CINT) {
					tok = TOK_CUINT;
				} else if (tok == TOK_CLLONG) {
					tok = TOK_CULLONG;
				}
				ch = *p++;
			} else {
				break;
			}
		}
		if (tok == TOK_CINT || tok == TOK_CUINT) {
			tokc.ui = n;
		} else {
			tokc.ull = n;
		}
	}
	if (ch) {
		tcc_error ("invalid number\n");
	}
}


#define PARSE2(c1, tok1, c2, tok2)\
case c1:			\
	PEEKC (c, p);		\
	if (c == c2) {		\
		p++;		\
		tok = tok2;	\
	} else {		\
		tok = tok1;	\
	}			\
	break;

/* return next token without macro substitution */
static inline void next_nomacro1(void)
{
	int t, c, is_long;
	TokenSym *ts;
	uint8_t *p, *p1;
	unsigned int h;

	p = file->buf_ptr;
redo_no_start:
	c = *p;
	switch (c) {
	case ' ':
	case '\t':
		tok = c;
		p++;
		goto keep_tok_flags;
	case '\f':
	case '\v':
	case '\r':
		p++;
		goto redo_no_start;
	case '\\':
		/* first look if it is in fact an end of buffer */
		if (p >= file->buf_end) {
			file->buf_ptr = p;
			handle_eob ();
			p = file->buf_ptr;
			if (p >= file->buf_end) {
				goto parse_eof;
			} else {
				goto redo_no_start;
			}
		} else {
			file->buf_ptr = p;
			ch = *p;
			handle_stray ();
			p = file->buf_ptr;
			goto redo_no_start;
		}
parse_eof:
		{
			TCCState *s1 = tcc_state;
			if ((parse_flags & PARSE_FLAG_LINEFEED)
			    && !(tok_flags & TOK_FLAG_EOF)) {
				tok_flags |= TOK_FLAG_EOF;
				tok = TOK_LINEFEED;
				goto keep_tok_flags;
			} else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
				tok = TOK_EOF;
			} else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
				tcc_error ("missing #endif");
			} else if (s1->include_stack_ptr == s1->include_stack) {
				/* no include left : end of file. */
				tok = TOK_EOF;
			} else {
				tok_flags &= ~TOK_FLAG_EOF;
				/* pop include file */

				/* test if previous '#endif' was after a #ifdef at
				   start of file */
				if (tok_flags & TOK_FLAG_ENDIF) {
#ifdef INC_DEBUG
					printf ("#endif %s\n", get_tok_str (file->ifndef_macro_saved, NULL));
#endif
					add_cached_include (s1, file->filename, file->ifndef_macro_saved);
					tok_flags &= ~TOK_FLAG_ENDIF;
				}

				/* pop include stack */
				tcc_close ();
				s1->include_stack_ptr--;
				p = file->buf_ptr;
				goto redo_no_start;
			}
		}
		break;

	case '\n':
		file->line_num++;
		tok_flags |= TOK_FLAG_BOL;
		p++;
maybe_newline:
		if (0 == (parse_flags & PARSE_FLAG_LINEFEED)) {
			goto redo_no_start;
		}
		tok = TOK_LINEFEED;
		goto keep_tok_flags;

	case '#':
		/* XXX: simplify */
		PEEKC (c, p);
		if ((tok_flags & TOK_FLAG_BOL) &&
		    (parse_flags & PARSE_FLAG_PREPROCESS)) {
			file->buf_ptr = p;
			preprocess (tok_flags & TOK_FLAG_BOF);
			p = file->buf_ptr;
			goto maybe_newline;
		} else {
			if (c == '#') {
				p++;
				tok = TOK_TWOSHARPS;
			} else {
				if (parse_flags & PARSE_FLAG_ASM_COMMENTS) {
					p = parse_line_comment (p - 1);
					goto redo_no_start;
				} else {
					tok = '#';
				}
			}
		}
		break;

	case 'a': case 'b': case 'c': case 'd':
	case 'e': case 'f': case 'g': case 'h':
	case 'i': case 'j': case 'k': case 'l':
	case 'm': case 'n': case 'o': case 'p':
	case 'q': case 'r': case 's': case 't':
	case 'u': case 'v': case 'w': case 'x':
	case 'y': case 'z':
	case 'A': case 'B': case 'C': case 'D':
	case 'E': case 'F': case 'G': case 'H':
	case 'I': case 'J': case 'K':
	case 'M': case 'N': case 'O': case 'P':
	case 'Q': case 'R': case 'S': case 'T':
	case 'U': case 'V': case 'W': case 'X':
	case 'Y': case 'Z':
	case '_': case '.':
parse_ident_fast:
		p1 = p;
		h = TOK_HASH_INIT;
		h = TOK_HASH_FUNC (h, c);
		p++;
		for (;;) {
			c = *p;
			if (!isidnum_table[*p - CH_EOF]) {
				break;
			}
			// dot handling here too
			if (isdot (c)) {
				PEEKC (c, p);
				if (isnum (c)) {
					cstr_reset (&tokcstr);
					cstr_ccat (&tokcstr, '.');
					goto parse_num;
				} else if (isdot (c)) {
					goto parse_dots;
				}
			}
			h = TOK_HASH_FUNC (h, *p);
			p++;
		}
		if (c != '\\') {
			TokenSym **pts;
			int len;

			/* fast case : no stray found, so we have the full token
			   and we have already hashed it */
			len = p - p1;
			h &= (TOK_HASH_SIZE - 1);
			pts = &hash_ident[h];
			for (;;) {
				ts = *pts;
				if (!ts) {
					break;
				}
				if (ts->len == len && !memcmp (ts->str, p1, len)) {
					goto token_found;
				}
				pts = &(ts->hash_next);
			}
			ts = tok_alloc_new (pts, (const char *) p1, len);
token_found:
			;
		} else {
			/* slower case */
			cstr_reset (&tokcstr);

			while (p1 < p) {
				cstr_ccat (&tokcstr, *p1);
				p1++;
			}
			p--;
			PEEKC (c, p);
parse_ident_slow:
			while (isidnum_table[((c > 255)? 255: c) - CH_EOF]) {
				cstr_ccat (&tokcstr, c);
				PEEKC (c, p);
			}
			ts = tok_alloc (tokcstr.data, tokcstr.size);
		}
		tok = ts->tok;
		break;
	case 'L':
		t = p[1];
		if (t != '\\' && t != '\'' && t != '\"') {
			/* fast case */
			goto parse_ident_fast;
		} else {
			PEEKC (c, p);
			if (c == '\'' || c == '\"') {
				is_long = 1;
				goto str_const;
			} else {
				cstr_reset (&tokcstr);
				cstr_ccat (&tokcstr, 'L');
				goto parse_ident_slow;
			}
		}
		break;
	case '0': case '1': case '2': case '3':
	case '4': case '5': case '6': case '7':
	case '8': case '9':

		cstr_reset (&tokcstr);
		/* after the first digit, accept digits, alpha, '.' or sign if
		   prefixed by 'eEpP' */
parse_num:
		for (;;) {
			t = c;
			cstr_ccat (&tokcstr, c);
			PEEKC (c, p);
			if (!(isnum (c) || isid (c) || isdot (c) ||
			      ((c == '+' || c == '-') &&
			       (t == 'e' || t == 'E' || t == 'p' || t == 'P')))) {
				break;
			}
		}
		/* We add a trailing '\0' to ease parsing */
		cstr_ccat (&tokcstr, '\0');
		tokc.cstr = &tokcstr;
		tok = TOK_PPNUM;
		break;
		/* special dot handling because it can also start a number */
parse_dots:
		if (!isdot (c)) {
			expect ("'.'");
			return;
		}
		PEEKC (c, p);
		tok = TOK_DOTS;
		break;
	case '\'':
	case '\"':
		is_long = 0;
str_const:
		{
			CString str;
			int sep;

			sep = c;

			/* parse the string */
			cstr_new (&str);
			p = parse_pp_string (p, sep, &str);
			if (!p) {
				return;
			}
			cstr_ccat (&str, '\0');

			/* eval the escape (should be done as TOK_PPNUM) */
			cstr_reset (&tokcstr);
			parse_escape_string (&tokcstr, str.data, is_long);
			cstr_free (&str);

			if (sep == '\'') {
				int char_size;
				/* XXX: make it portable */
				if (!is_long) {
					char_size = 1;
				} else {
					char_size = sizeof(nwchar_t);
				}
				if (tokcstr.size <= char_size) {
					tcc_error ("empty character constant");
				}
				if (tokcstr.size > 2 * char_size) {
					tcc_warning ("multi-character character constant");
				}
				if (!is_long) {
					tokc.i = *(int8_t *) tokcstr.data;
					tok = TOK_CCHAR;
				} else {
					tokc.i = *(nwchar_t *) tokcstr.data;
					tok = TOK_LCHAR;
				}
			} else {
				tokc.cstr = &tokcstr;
				if (!is_long) {
					tok = TOK_STR;
				} else {
					tok = TOK_LSTR;
				}
			}
		}
		break;

	case '<':
		PEEKC (c, p);
		if (c == '=') {
			p++;
			tok = TOK_LE;
		} else if (c == '<') {
			PEEKC (c, p);
			if (c == '=') {
				p++;
				tok = TOK_A_SHL;
			} else {
				tok = TOK_SHL;
			}
		} else {
			tok = TOK_LT;
		}
		break;

	case '>':
		PEEKC (c, p);
		if (c == '=') {
			p++;
			tok = TOK_GE;
		} else if (c == '>') {
			PEEKC (c, p);
			if (c == '=') {
				p++;
				tok = TOK_A_SAR;
			} else {
				tok = TOK_SAR;
			}
		} else {
			tok = TOK_GT;
		}
		break;

	case '&':
		PEEKC (c, p);
		if (c == '&') {
			p++;
			tok = TOK_LAND;
		} else if (c == '=') {
			p++;
			tok = TOK_A_AND;
		} else {
			tok = '&';
		}
		break;

	case '|':
		PEEKC (c, p);
		if (c == '|') {
			p++;
			tok = TOK_LOR;
		} else if (c == '=') {
			p++;
			tok = TOK_A_OR;
		} else {
			tok = '|';
		}
		break;

	case '+':
		PEEKC (c, p);
		if (c == '+') {
			p++;
			tok = TOK_INC;
		} else if (c == '=') {
			p++;
			tok = TOK_A_ADD;
		} else {
			tok = '+';
		}
		break;

	case '-':
		PEEKC (c, p);
		if (c == '-') {
			p++;
			tok = TOK_DEC;
		} else if (c == '=') {
			p++;
			tok = TOK_A_SUB;
		} else if (c == '>') {
			p++;
			tok = TOK_ARROW;
		} else {
			tok = '-';
		}
		break;

		PARSE2 ('!', '!', '=', TOK_NE)
		PARSE2 ('=', '=', '=', TOK_EQ)
		PARSE2 ('*', '*', '=', TOK_A_MUL)
		PARSE2 ('%', '%', '=', TOK_A_MOD)
		PARSE2 ('^', '^', '=', TOK_A_XOR)

	/* comments or operator */
	case '/':
		PEEKC (c, p);
		if (c == '*') {
			p = parse_comment (p);
			/* comments replaced by a blank */
			tok = ' ';
			goto keep_tok_flags;
		} else if (c == '/') {
			p = parse_line_comment (p);
			tok = ' ';
			goto keep_tok_flags;
		} else if (c == '=') {
			p++;
			tok = TOK_A_DIV;
		} else {
			tok = '/';
		}
		break;

	/* simple tokens */
	case '(':
	case ')':
	case '[':
	case ']':
	case '{':
	case '}':
	case ',':
	case ';':
	case ':':
	case '?':
	case '~':
	case '$':	/* only used in assembler */
	case '@':	/* dito */
		tok = c;
		p++;
		break;
	default:
		tcc_error ("unrecognized character \\x%02x", c);
		break;
	}
	tok_flags = 0;
keep_tok_flags:
	file->buf_ptr = p;
#if defined(PARSE_DEBUG)
	printf ("token = %s\n", get_tok_str (tok, &tokc));
#endif
}

/* return next token without macro substitution. Can read input from
   macro_ptr buffer */
static void next_nomacro_spc(void)
{
	if (macro_ptr) {
redo:
		tok = *macro_ptr;
		if (tok) {
			TOK_GET (&tok, &macro_ptr, &tokc);
			if (tok == TOK_LINENUM) {
				file->line_num = tokc.i;
				goto redo;
			}
		}
	} else {
		next_nomacro1 ();
	}
}

ST_FUNC void next_nomacro(void)
{
	do {
		next_nomacro_spc ();
	} while (tcc_nerr () == 0 && is_space (tok));
}

/* substitute args in macro_str and return allocated string */
static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
{
	int last_tok, t, spc;
	const int *st;
	Sym *s;
	CValue cval;
	TokenString str;
	CString cstr;

	tok_str_new (&str);
	last_tok = 0;
	while (tcc_nerr () == 0) {
		TOK_GET (&t, &macro_str, &cval);
		if (!t) {
			break;
		}
		if (t == '#') {
			/* stringize */
			TOK_GET (&t, &macro_str, &cval);
			if (!t) {
				break;
			}
			s = sym_find2 (args, t);
			if (s) {
				cstr_new (&cstr);
				st = s->d;
				spc = 0;
				while (*st) {
					TOK_GET (&t, &st, &cval);
					if (!check_space (t, &spc)) {
						cstr_cat (&cstr, get_tok_str (t, &cval));
					}
				}
				cstr.size -= spc;
				cstr_ccat (&cstr, '\0');
#ifdef PP_DEBUG
				printf ("stringize: %s\n", (char *) cstr.data);
#endif
				/* add string */
				cval.cstr = &cstr;
				tok_str_add2 (&str, TOK_STR, &cval);
				cstr_free (&cstr);
			} else {
				tok_str_add2 (&str, t, &cval);
			}
		} else if (t >= TOK_IDENT) {
			s = sym_find2 (args, t);
			if (s) {
				st = s->d;
				/* if '##' is present before or after, no arg substitution */
				if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
					/* special case for var arg macros : ## eats the
					   ',' if empty VA_ARGS variable. */
					/* XXX: test of the ',' is not 100%
					   reliable. should fix it to avoid security
					   problems */
					if (gnu_ext && s->type.t &&
					    last_tok == TOK_TWOSHARPS &&
					    str.len >= 2 && str.str[str.len - 2] == ',') {
						if (*st == 0) {
							/* suppress ',' '##' */
							str.len -= 2;
						} else {
							/* suppress '##' and add variable */
							str.len--;
							goto add_var;
						}
					} else {
						int t1;
add_var:
						for (;;) {
							TOK_GET (&t1, &st, &cval);
							if (!t1) {
								break;
							}
							tok_str_add2 (&str, t1, &cval);
						}
					}
				} else {
					/* NOTE: the stream cannot be read when macro
					   substituing an argument */
					macro_subst (&str, nested_list, st, NULL);
				}
			} else {
				tok_str_add (&str, t);
			}
		} else {
			tok_str_add2 (&str, t, &cval);
		}
		last_tok = t;
	}
	tok_str_add (&str, 0);
	return str.str;
}

static char const ab_month_name[12][4] =
{
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

/* do macro substitution of current token with macro 's' and add
   result to (tok_str,tok_len). 'nested_list' is the list of all
   macros we got inside to avoid recursing. Return non zero if no
   substitution needs to be done */
static int macro_subst_tok(TokenString *tok_str,
			   Sym **nested_list, Sym *s, struct macro_level **can_read_stream)
{
	Sym *args, *sa, *sa1;
	int mstr_allocated, parlevel, *mstr, t, t1, spc;
	const int *p;
	TokenString str;
	char *cstrval;
	CValue cval;
	CString cstr;
	char buf[32];

	/* if symbol is a macro, prepare substitution */
	/* special macros */
	if (tok == TOK___LINE__) {
		snprintf (buf, sizeof(buf), "%d", file->line_num);
		cstrval = buf;
		t1 = TOK_PPNUM;
		goto add_cstr1;
	} else if (tok == TOK___FILE__) {
		cstrval = file->filename;
		goto add_cstr;
	} else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
		time_t ti;
		struct tm *tm;

		time (&ti);
		tm = localtime (&ti);
		if (tok == TOK___DATE__) {
			snprintf (buf, sizeof(buf), "%s %2d %d",
				ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
		} else {
			snprintf (buf, sizeof(buf), "%02d:%02d:%02d",
				tm->tm_hour, tm->tm_min, tm->tm_sec);
		}
		cstrval = buf;
add_cstr:
		t1 = TOK_STR;
add_cstr1:
		cstr_new (&cstr);
		cstr_cat (&cstr, cstrval);
		cstr_ccat (&cstr, '\0');
		cval.cstr = &cstr;
		tok_str_add2 (tok_str, t1, &cval);
		cstr_free (&cstr);
	} else {
		mstr = s->d;
		mstr_allocated = 0;
		if (s->type.t == MACRO_FUNC) {
			/* NOTE: we do not use next_nomacro to avoid eating the
			   next token. XXX: find better solution */
redo:
			if (macro_ptr) {
				p = macro_ptr;
				while (is_space (t = *p) || TOK_LINEFEED == t)
					++p;
				if (t == 0 && can_read_stream) {
					/* end of macro stream: we must look at the token
					   after in the file */
					struct macro_level *ml = *can_read_stream;
					macro_ptr = NULL;
					if (ml) {
						macro_ptr = ml->p;
						ml->p = NULL;
						*can_read_stream = ml->prev;
					}
					/* also, end of scope for nested defined symbol */
					(*nested_list)->v = -1;
					goto redo;
				}
			} else {
				ch = file->buf_ptr[0];
				while (is_space (ch) || ch == '\n' || ch == '/') {
					if (ch == '/') {
						int c;
						uint8_t *p = file->buf_ptr;
						PEEKC (c, p);
						if (c == '*') {
							p = parse_comment (p);
							file->buf_ptr = p - 1;
						} else if (c == '/') {
							p = parse_line_comment (p);
							file->buf_ptr = p - 1;
						} else {
							break;
						}
					}
					cinp ();
				}
				t = ch;
			}
			if (t != '(') {	/* no macro subst */
				return -1;
			}

			/* argument macro */
			next_nomacro ();
			next_nomacro ();
			args = NULL;
			sa = s->next;
			/* NOTE: empty args are allowed, except if no args */
			while (tcc_nerr () == 0) {
				/* handle '()' case */
				if (!args && !sa && tok == ')') {
					break;
				}
				if (!sa) {
					tcc_error ("macro '%s' used with too many args",
						get_tok_str (s->v, 0));
				}
				tok_str_new (&str);
				parlevel = spc = 0;
				/* NOTE: non zero sa->t indicates VA_ARGS */
				while ((parlevel > 0 ||
					(tok != ')' &&
					 (tok != ',' || (sa && sa->type.t)))) &&
				       tok != -1) {
					if (tok == '(') {
						parlevel++;
					} else if (tok == ')') {
						parlevel--;
					}
					if (tok == TOK_LINEFEED) {
						tok = ' ';
					}
					if (!check_space (tok, &spc)) {
						tok_str_add2 (&str, tok, &tokc);
					}
					next_nomacro_spc ();
				}
				str.len -= spc;
				tok_str_add (&str, 0);
				sa1 = sa ? sym_push2 (&args, sa->v & ~SYM_FIELD, sa->type.t, 0) : NULL;
				if (!sa1) {
					return -1;
				}
				sa1->d = str.str;
				sa = sa->next;
				if (tok == ')') {
					/* special case for gcc var args: add an empty
					   var arg argument if it is omitted */
					if (sa && sa->type.t && gnu_ext) {
						continue;
					} else {
						break;
					}
				}
				if (tok != ',') {
					expect (",");
					return 1;
				}
				next_nomacro ();
			}
			if (sa) {
				tcc_error ("macro '%s' used with too few args",
					get_tok_str (s->v, 0));
			}

			/* now subst each arg */
			mstr = macro_arg_subst (nested_list, mstr, args);
			/* free memory */
			sa = args;
			while (sa) {
				sa1 = sa->prev;
				tok_str_free (sa->d);
				sym_free (sa);
				sa = sa1;
			}
			mstr_allocated = 1;
		}
		if (sym_push2 (nested_list, s->v, 0, 0) == 0) {
			return -1;
		}
		macro_subst (tok_str, nested_list, mstr, can_read_stream);
		/* pop nested defined symbol */
		sa1 = *nested_list;
		*nested_list = sa1->prev;
		sym_free (sa1);
		if (mstr_allocated) {
			tok_str_free (mstr);
		}
	}
	return 0;
}

/* handle the '##' operator. Return NULL if no '##' seen. Otherwise
   return the resulting string (which must be freed). */
static inline int *macro_twosharps(const int *macro_str)
{
	const int *ptr;
	int t;
	TokenString macro_str1;
	CString cstr;
	int n, start_of_nosubsts;

	/* we search the first '##' */
	for (ptr = macro_str;;) {
		CValue cval;
		TOK_GET (&t, &ptr, &cval);
		if (t == TOK_TWOSHARPS) {
			break;
		}
		/* nothing more to do if end of string */
		if (t == 0) {
			return NULL;
		}
	}

	/* we saw '##', so we need more processing to handle it */
	start_of_nosubsts = -1;
	tok_str_new (&macro_str1);
	for (ptr = macro_str;;) {
		TOK_GET (&tok, &ptr, &tokc);
		if (tok == 0) {
			break;
		}
		if (tok == TOK_TWOSHARPS) {
			continue;
		}
		if (tok == TOK_NOSUBST && start_of_nosubsts < 0) {
			start_of_nosubsts = macro_str1.len;
		}
		while (*ptr == TOK_TWOSHARPS) {
			/* given 'a##b', remove nosubsts preceding 'a' */
			if (start_of_nosubsts >= 0) {
				macro_str1.len = start_of_nosubsts;
			}
			/* given 'a##b', skip '##' */
			t = *++ptr;
			/* given 'a##b', remove nosubsts preceding 'b' */
			while (t == TOK_NOSUBST)
				t = *++ptr;
			if (t && t != TOK_TWOSHARPS) {
				CValue cval;
				TOK_GET (&t, &ptr, &cval);
				/* We concatenate the two tokens */
				cstr_new (&cstr);
				cstr_cat (&cstr, get_tok_str (tok, &tokc));
				n = cstr.size;
				cstr_cat (&cstr, get_tok_str (t, &cval));
				cstr_ccat (&cstr, '\0');

				tcc_open_bf (tcc_state, ":paste:", cstr.size);
				memcpy (file->buffer, cstr.data, cstr.size);
				while (tcc_nerr () == 0) {
					next_nomacro1 ();
					if (0 == *file->buf_ptr) {
						break;
					}
					tok_str_add2 (&macro_str1, tok, &tokc);
					tcc_warning ("pasting \"%.*s\" and \"%s\" does not give a valid preprocessing token",
						n, (char *) cstr.data, (char *) cstr.data + n);
				}
				tcc_close ();
				cstr_free (&cstr);
			}
		}
		if (tok != TOK_NOSUBST) {
			start_of_nosubsts = -1;
		}
		tok_str_add2 (&macro_str1, tok, &tokc);
	}
	tok_str_add (&macro_str1, 0);
	return macro_str1.str;
}


/* do macro substitution of macro_str and add result to
   (tok_str,tok_len). 'nested_list' is the list of all macros we got
   inside to avoid recursing. */
static void macro_subst(TokenString *tok_str, Sym **nested_list,
			const int *macro_str, struct macro_level **can_read_stream)
{
	Sym *s;
	int *macro_str1;
	const int *ptr;
	int t, ret, spc;
	CValue cval;
	struct macro_level ml;
	int force_blank;

	/* first scan for '##' operator handling */
	ptr = macro_str;
	macro_str1 = macro_twosharps (ptr);

	if (macro_str1) {
		ptr = macro_str1;
	}
	spc = 0;
	force_blank = 0;

	while (tcc_nerr () == 0) {
		/* NOTE: ptr == NULL can only happen if tokens are read from
		   file stream due to a macro function call */
		if (ptr == NULL) {
			break;
		}
		TOK_GET (&t, &ptr, &cval);
		if (t == 0) {
			break;
		}
		if (t == TOK_NOSUBST) {
			/* following token has already been subst'd. just copy it on */
			tok_str_add2 (tok_str, TOK_NOSUBST, NULL);
			TOK_GET (&t, &ptr, &cval);
			goto no_subst;
		}
		s = define_find (t);
		if (s != NULL) {
			/* if nested substitution, do nothing */
			if (sym_find2 (*nested_list, t)) {
				/* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
				tok_str_add2 (tok_str, TOK_NOSUBST, NULL);
				goto no_subst;
			}
			ml.p = macro_ptr;
			if (can_read_stream) {
				ml.prev = *can_read_stream, *can_read_stream = &ml;
			}
			macro_ptr = (int *) ptr;
			tok = t;
			ret = macro_subst_tok (tok_str, nested_list, s, can_read_stream);
			ptr = (int *) macro_ptr;
			macro_ptr = ml.p;
			if (can_read_stream && *can_read_stream == &ml) {
				*can_read_stream = ml.prev;
			}
			if (ret != 0) {
				goto no_subst;
			}
			if (parse_flags & PARSE_FLAG_SPACES) {
				force_blank = 1;
			}
		} else {
no_subst:
			if (force_blank) {
				tok_str_add (tok_str, ' ');
				spc = 1;
				force_blank = 0;
			}
			if (!check_space (t, &spc)) {
				tok_str_add2 (tok_str, t, &cval);
			}
		}
	}
	if (macro_str1) {
		tok_str_free (macro_str1);
	}
}

/* return next token with macro substitution */
ST_FUNC void next(void)
{
	Sym *nested_list, *s;
	TokenString str;
	struct macro_level *ml;

redo:
	if (parse_flags & PARSE_FLAG_SPACES) {
		next_nomacro_spc ();
	} else {
		next_nomacro ();
	}
	if (!macro_ptr) {
		/* if not reading from macro substituted string, then try
		   to substitute macros */
		if (tok >= TOK_IDENT &&
		    (parse_flags & PARSE_FLAG_PREPROCESS)) {
			s = define_find (tok);
			if (s) {
				/* we have a macro: we try to substitute */
				tok_str_new (&str);
				nested_list = NULL;
				ml = NULL;
				if (macro_subst_tok (&str, &nested_list, s, &ml) == 0) {
					/* substitution done, NOTE: maybe empty */
					tok_str_add (&str, 0);
					macro_ptr = str.str;
					macro_ptr_allocated = str.str;
					goto redo;
				}
			}
		}
	} else {
		if (tok == 0) {
			/* end of macro or end of unget buffer */
			if (unget_buffer_enabled) {
				macro_ptr = unget_saved_macro_ptr;
				unget_buffer_enabled = 0;
			} else {
				/* end of macro string: free it */
				tok_str_free (macro_ptr_allocated);
				macro_ptr_allocated = NULL;
				macro_ptr = NULL;
			}
			goto redo;
		} else if (tok == TOK_NOSUBST) {
			/* discard preprocessor's nosubst markers */
			goto redo;
		}
	}

	/* convert preprocessor tokens into C tokens */
	if (tok == TOK_PPNUM &&
	    (parse_flags & PARSE_FLAG_TOK_NUM)) {
		parse_number ((char *) tokc.cstr->data);
	}
}

/* push back current token and set current token to 'last_tok'. Only
   identifier case handled for labels. */
ST_INLN void unget_tok(int last_tok)
{
	int i, n;
	int *q;
	if (unget_buffer_enabled) {
		/* assert(macro_ptr == unget_saved_buffer + 1);
		   assert(*macro_ptr == 0);  */
	} else {
		unget_saved_macro_ptr = macro_ptr;
		unget_buffer_enabled = 1;
	}
	q = unget_saved_buffer;
	macro_ptr = q;
	*q++ = tok;
	n = tok_ext_size (tok) - 1;
	for (i = 0; i < n; i++) {
		*q++ = tokc.tab[i];
	}
	*q = 0;	/* end of token string */
	tok = last_tok;
}


/* better than nothing, but needs extension to handle '-E' option
   correctly too */
ST_FUNC void preprocess_init(TCCState *s1)
{
	s1->include_stack_ptr = s1->include_stack;
	/* XXX: move that before to avoid having to initialize
	   file->ifdef_stack_ptr ? */
	s1->ifdef_stack_ptr = s1->ifdef_stack;
	file->ifdef_stack_ptr = s1->ifdef_stack_ptr;

	vtop = vstack - 1;
	s1->pack_stack[0] = 0;
	s1->pack_stack_ptr = s1->pack_stack;
}

ST_FUNC void preprocess_new(void)
{
	int i, c;
	const char *p, *r;

	/* init isid table */
	for (i = CH_EOF; i < 256; i++) {
		isidnum_table[i - CH_EOF] = isid (i) || isnum (i) || isdot (i);
	}

	/* add all tokens */
	table_ident = NULL;
	memset (hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));

	tok_ident = TOK_IDENT;
	p = tcc_keywords;
	while (*p) {
		r = p;
		for (;;) {
			c = *r++;
			if (c == '\0') {
				break;
			}
		}
		tok_alloc (p, r - p - 1);
		p = r;
	}
}

/* Preprocess the current file */
ST_FUNC int tcc_preprocess(TCCState *s1)
{
	Sym *define_start;

	BufferedFile *file_ref, **iptr, **iptr_new;
	int token_seen, line_ref, d;
	const char *s;

	preprocess_init (s1);
	define_start = define_stack;
	ch = file->buf_ptr[0];
	tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
	parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
		      PARSE_FLAG_LINEFEED | PARSE_FLAG_SPACES;
	token_seen = 0;
	line_ref = 0;
	file_ref = NULL;
	iptr = s1->include_stack_ptr;

	while (tcc_nerr () == 0) {
		next ();
		if (tok == TOK_EOF) {
			break;
		} else if (file != file_ref) {
			goto print_line;
		} else if (tok == TOK_LINEFEED) {
			if (!token_seen) {
				continue;
			}
			++line_ref;
			token_seen = 0;
		} else if (!token_seen) {
			d = file->line_num - line_ref;
			if (file != file_ref || d < 0 || d >= 8) {
print_line:
				iptr_new = s1->include_stack_ptr;
				s = iptr_new > iptr? " 1"
				    : iptr_new < iptr? " 2"
				    : iptr_new > s1->include_stack? " 3"
				    : ""
				;
				iptr = iptr_new;
				fprintf (s1->ppfp, "# %d \"%s\"%s\n", file->line_num, file->filename, s);
			} else {
				while (d)
					fputs ("\n", s1->ppfp), --d;
			}
			line_ref = (file_ref = file)->line_num;
			token_seen = tok != TOK_LINEFEED;
			if (!token_seen) {
				continue;
			}
		}
		fputs (get_tok_str (tok, &tokc), s1->ppfp);
	}
	free_defines (define_start);
	return 0;
}