diff -r 6e704fc09528 -r e14a1aba4c56 tokenizer.c
--- a/tokenizer.c	Sat Feb 19 23:23:29 2000 +0100
+++ b/tokenizer.c	Wed Feb 23 08:47:59 2000 +0100
@@ -41,8 +41,8 @@
 	DUMMY_TR("Too few arguments"),					/* E_TOKZ_TOO_FEW_ARGS */
 	DUMMY_TR("Too many arguments"),					/* E_TOKZ_TOO_MANY_ARGS */
 	DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */
-	DUMMY_TR("Unexpected end of statement"),		/* E_TOKZ_UNEXPECTED_EOS */
 	DUMMY_TR("Identifier expected"),				/* E_TOKZ_IDENTIFIER_EXPECTED */
+	DUMMY_TR("Starting brace ('{') expected"),		/* E_TOKZ_LBRACE_EXPECTED */
 };
 
 
@@ -50,11 +50,11 @@
 
 #define STRBLEN 32
 
-#define STRING_DECL(X) char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0
-#define STRING_DECL_P(X, P) char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1
-#define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) return -ENOMEM;}
+#define STRING_DECL(X) int err=0; char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0
+#define STRING_DECL_P(X, P) int err=0; char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1
+#define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) err=-ENOMEM;}
 #define STRING_FREE(X) if(X!=NULL) free(X)
-#define STRING_FINISH(X) {if(!_string_finish(&X, X##_tmp, X##_tmpl)) return -ENOMEM;}
+#define STRING_FINISH(X) {if(err!=0) return err; if(!_string_finish(&X, X##_tmp, X##_tmpl)) err=-ENOMEM;}
 
 
 static bool _string_append(char **p, char *tmp, int *tmplen, char c)
@@ -575,6 +575,12 @@
 	
 	tok_free(tok);
 	
+	if(!TOK_IS_INVALID(&(tokz->ungettok))){
+		*tok=tokz->ungettok;
+		tokz->ungettok.type=TOK_INVALID;
+		return TRUE;
+	}
+
 	while(1){
 	
 		e=0;
@@ -607,9 +613,11 @@
 					TOK_SET_OP(tok, OP_EOF);
 					return FALSE;
 				}
-				if(!isspace(c)){
-					tokz_warn_error(tokz, tokz->line, E_TOKZ_EOL_EXPECTED);
-					return FALSE;
+				if(!isspace(c) && e==0){
+					e=E_TOKZ_EOL_EXPECTED;
+					tokz_warn_error(tokz, tokz->line, e);
+					if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
+						return FALSE;
 				}
 			}while(c!='\n');
 			
@@ -627,30 +635,28 @@
 			continue;
 			
 		case '/':
-			{
-				c2=GETCH();
-				
-				if(c2=='='){
-					TOK_SET_OP(tok, OP_AS_DIV);
-					return TRUE;
-				}
-				
-				if(c2!='*'){
-					UNGETCH(c2);
-					TOK_SET_OP(tok, OP_DIV);
-					return TRUE;
-				}
-
-				if(tokz->flags&TOKZ_READ_COMMENTS){
-					e=scan_c_comment(tok, tokz);
-					break;
-				}else if((e=skip_c_comment(tokz))){
-					break;
-				}
-				
-				continue;
+			c2=GETCH();
+			
+			if(c2=='='){
+				TOK_SET_OP(tok, OP_AS_DIV);
+				return TRUE;
 			}
 			
+			if(c2!='*'){
+				UNGETCH(c2);
+				TOK_SET_OP(tok, OP_DIV);
+				return TRUE;
+			}
+			
+			if(tokz->flags&TOKZ_READ_COMMENTS){
+				e=scan_c_comment(tok, tokz);
+				break;
+			}else if((e=skip_c_comment(tokz))){
+				break;
+			}
+			
+			continue;
+			
 		case '\"':
 			e=scan_string(tok, tokz, TRUE);
 			break;
@@ -680,6 +686,14 @@
 }
 
 
+void tokz_unget_token(Tokenizer *tokz, Token *tok)
+{
+	tok_free(&(tokz->ungettok));	
+	tokz->ungettok=*tok;
+	tok->type=TOK_INVALID;
+}
+
+
 /*
  * File open
  */
@@ -701,7 +715,8 @@
 	finfo->name=tokz->name;
 	finfo->line=tokz->line;
 	finfo->ungetc=tokz->ungetc;
-
+	finfo->ungettok=tokz->ungettok;
+	
 	return TRUE;
 }
 
@@ -721,7 +736,8 @@
 	tokz->file=file;
 	tokz->name=NULL;
 	tokz->line=1;
-	tokz->ungetc=-1;
+	tokz->ungetc=-1;	
+	tokz->ungettok.type=TOK_INVALID;
 	
 	return TRUE;
 }
@@ -776,7 +792,8 @@
 	tokz->file=NULL;
 	tokz->name=NULL;
 	tokz->line=1;
-	tokz->ungetc=-1;
+	tokz->ungetc=-1;	
+	tokz->ungettok.type=TOK_INVALID;
 	tokz->flags=0;
 	tokz->optstack=NULL;
 	tokz->nest_lvl=0;
@@ -839,7 +856,8 @@
 	tokz->name=finfo->name;
 	tokz->line=finfo->line;
 	tokz->ungetc=finfo->ungetc;
-
+	tokz->ungettok=finfo->ungettok;
+	
 	if(tokz->filestack_n==0){
 		free(tokz->filestack);
 		tokz->filestack=NULL;
@@ -871,7 +889,8 @@
 		fclose(tokz->file);
 	if(tokz->name!=NULL)
 		free(tokz->name);
-
+	tok_free(&(tokz->ungettok));
+	
 	free(tokz);
 }
 
@@ -882,8 +901,10 @@
 
 void tok_free(Token *tok)
 {
-	if(TOK_IS_STRING(tok))
-		free(TOK_STRING_VAL(tok));
+	if(TOK_IS_STRING(tok) || TOK_IS_IDENT(tok) || TOK_IS_COMMENT(tok)){
+		if(TOK_STRING_VAL(tok)!=NULL)
+			free(TOK_STRING_VAL(tok));
+	}
 	
 	tok->type=TOK_INVALID;
 }