tokenizer.c

Mon, 20 Apr 2020 10:14:32 -0500

author
Tuomo Valkonen <tuomov@iki.fi>
date
Mon, 20 Apr 2020 10:14:32 -0500
changeset 119
87e3bb5086e8
parent 79
81e9a32b1c0c
permissions
-rw-r--r--

Convert README to markdown

0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
1 /*
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
2 * libtu/tokenizer.c
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
3 *
36
63cd573ffbcf trunk: changeset 39
tuomov
parents: 35
diff changeset
4 * Copyright (c) Tuomo Valkonen 1999-2002.
53
f8f9366b359c trunk: changeset 57
tuomov
parents: 47
diff changeset
5 *
f8f9366b359c trunk: changeset 57
tuomov
parents: 47
diff changeset
6 * You may distribute and modify this library under the terms of either
f8f9366b359c trunk: changeset 57
tuomov
parents: 47
diff changeset
7 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
8 */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
9
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
10 #include <errno.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
11 #include <stdio.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
12 #include <ctype.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
13 #include <limits.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
14 #include <assert.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
15 #include <math.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
16 #include <string.h>
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
17
70
e866dc825481 trunk: changeset 1408
tuomov
parents: 62
diff changeset
18 #include "tokenizer.h"
e866dc825481 trunk: changeset 1408
tuomov
parents: 62
diff changeset
19 #include "misc.h"
e866dc825481 trunk: changeset 1408
tuomov
parents: 62
diff changeset
20 #include "output.h"
79
81e9a32b1c0c trunk: changeset 1689
tuomov
parents: 70
diff changeset
21 #include "private.h"
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
22
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
23
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
24 static const char *errors[]={
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
25 DUMMY_TR("(no error)"),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
26 DUMMY_TR("Unexpected end of file"), /* E_TOKZ_UNEXPECTED_EOF */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
27 DUMMY_TR("Unexpected end of line"), /* E_TOKZ_UNEXPECTED_EOL */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
28 DUMMY_TR("End of line expected"), /* E_TOKZ_EOL_EXPECTED */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
29 DUMMY_TR("Invalid character"), /* E_TOKZ_INVALID_CHAR*/
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
30 DUMMY_TR("Numeric constant too big"), /* E_TOKZ_TOOBIG */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
31 DUMMY_TR("Invalid numberic format"), /* E_TOKZ_NUMFMT */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
32 DUMMY_TR("Junk after numeric constant"), /* E_TOKZ_NUM_JUNK */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
33 DUMMY_TR("Not an integer"), /* E_TOKZ_NOTINT */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
34 DUMMY_TR("Numeric constant out of range"), /* E_TOKZ_RANGE */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
35 DUMMY_TR("Multi-character character constant"), /* E_TOKZ_MULTICHAR */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
36 DUMMY_TR("Token/statement limit reached"), /* E_TOKZ_TOKEN_LIMIT */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
37 DUMMY_TR("Unknown option"), /* E_TOKZ_UNKONWN_OPTION */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
38 DUMMY_TR("Syntax error"), /* E_TOKZ_SYNTAX */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
39 DUMMY_TR("Invalid argument"), /* E_TOKZ_INVALID_ARGUMENT */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
40 DUMMY_TR("End of statement expected"), /* E_TOKZ_EOS_EXPECTED */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
41 DUMMY_TR("Too few arguments"), /* E_TOKZ_TOO_FEW_ARGS */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
42 DUMMY_TR("Too many arguments"), /* E_TOKZ_TOO_MANY_ARGS */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
43 DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
44 DUMMY_TR("Identifier expected"), /* E_TOKZ_IDENTIFIER_EXPECTED */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
45 DUMMY_TR("Starting brace ('{') expected"), /* E_TOKZ_LBRACE_EXPECTED */
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
46 };
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
47
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
48
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
49 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
50
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
51 #define STRBLEN 32
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
52
2
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
53 #define STRING_DECL(X) int err=0; char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
54 #define STRING_DECL_P(X, P) int err=0; char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
55 #define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) err=-ENOMEM;}
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
56 #define STRING_FREE(X) if(X!=NULL) free(X)
2
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
57 #define STRING_FINISH(X) {if(err!=0) return err; if(!_string_finish(&X, X##_tmp, X##_tmpl)) err=-ENOMEM;}
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
58
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
59
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
60 static bool _string_append(char **p, char *tmp, int *tmplen, char c)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
61 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
62 char *tmp2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
63
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
64 if(*tmplen==STRBLEN-1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
65 tmp[STRBLEN-1]='\0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
66 if(*p!=NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
67 tmp2=scat(*p, tmp);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
68 free(*p);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
69 *p=tmp2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
70 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
71 *p=scopy(tmp);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
72 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
73 *tmplen=1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
74 tmp[0]=c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
75 return *p!=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
76 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
77 tmp[(*tmplen)++]=c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
78 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
79 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
80 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
81
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
82
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
83 static bool _string_finish(char **p, char *tmp, int tmplen)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
84 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
85 char *tmp2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
86
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
87 if(tmplen==0){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
88 if(*p==NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
89 *p=scopy("");
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
90 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
91 tmp[tmplen]='\0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
92 if(*p!=NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
93 tmp2=scat(*p, tmp);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
94 free(*p);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
95 *p=tmp2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
96 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
97 *p=scopy(tmp);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
98 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
99 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
100 return *p!=NULL;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
101 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
102
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
103
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
104 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
105
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
106
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
107 #define INC_LINE() tokz->line++
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
108 #define GETCH() _getch(tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
109 #define UNGETCH(C) _ungetch(tokz, C)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
110
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
111 static int _getch(Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
112 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
113 int c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
114
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
115 if(tokz->ungetc!=-1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
116 c=tokz->ungetc;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
117 tokz->ungetc=-1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
118 }else if (tokz->flags&TOKZ_READ_FROM_BUFFER) {
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
119 assert(tokz->buffer.data!=NULL);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
120 if (tokz->buffer.pos==tokz->buffer.len)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
121 c=EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
122 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
123 c=tokz->buffer.data[tokz->buffer.pos++];
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
124 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
125 c=getc(tokz->file);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
126 }
20
3f11f764772a trunk: changeset 23
tuomov
parents: 14
diff changeset
127
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
128 return c;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
129 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
130
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
131
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
132 static void _ungetch(Tokenizer *tokz, int c)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
133 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
134 tokz->ungetc=c;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
135 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
136
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
137
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
138 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
139
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
140
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
141 static int scan_line_comment(Token *tok, Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
142 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
143 STRING_DECL_P(s, "#");
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
144 int c;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
145
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
146 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
147
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
148 while(c!='\n' && c!=EOF){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
149 STRING_APPEND(s, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
150 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
151 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
152
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
153 UNGETCH(c);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
154
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
155 STRING_FINISH(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
156
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
157 TOK_SET_COMMENT(tok, s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
158
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
159 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
160 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
161
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
162
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
163 static int skip_line_comment(Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
164 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
165 int c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
166
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
167 do{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
168 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
169 }while(c!='\n' && c!=EOF);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
170
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
171 UNGETCH(c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
172
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
173 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
174 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
175
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
176
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
177 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
178
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
179
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
180 static int scan_c_comment(Token *tok, Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
181 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
182 STRING_DECL_P(s, "/*");
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
183 int c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
184 int st=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
185
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
186 while(1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
187 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
188
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
189 if(c==EOF){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
190 STRING_FREE(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
191 return E_TOKZ_UNEXPECTED_EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
192 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
193
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
194 STRING_APPEND(s, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
195
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
196 if(c=='\n'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
197 INC_LINE();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
198 }else if(st==0 && c=='*'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
199 st=1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
200 }else if(st==1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
201 if(c=='/')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
202 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
203 st=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
204 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
205 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
206
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
207 STRING_FINISH(s);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
208
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
209 TOK_SET_COMMENT(tok, s);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
210
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
211 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
212 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
213
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
214
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
215 static int skip_c_comment(Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
216 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
217 int c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
218 int st=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
219
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
220 while(1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
221 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
222
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
223 if(c==EOF)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
224 return E_TOKZ_UNEXPECTED_EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
225
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
226 if(c=='\n')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
227 INC_LINE();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
228 else if(st==0 && c=='*')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
229 st=1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
230 else if(st==1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
231 if(c=='/')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
232 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
233 st=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
234 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
235 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
236
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
237 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
238 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
239
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
240
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
241 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
242
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
243
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
244 static int scan_char_escape(Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
245 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
246 static char* special_chars="nrtbae";
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
247 static char* specials="\n\r\t\b\a\033";
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
248 int base, max;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
249 int i ,c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
250
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
251 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
252
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
253 for(i=0;special_chars[i];i++){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
254 if(special_chars[i]==c)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
255 return specials[c];
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
256 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
257
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
258 if(c=='x' || c=='X'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
259 base=16;max=2;i=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
260 }else if(c=='d' || c=='D'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
261 base=10;max=3;i=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
262 }else if(c=='8' || c=='9'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
263 base=10;max=2;i=c-'0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
264 }else if('0'<=c && c<='7'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
265 base=8;max=2;i=c-'0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
266 }else if(c=='\n'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
267 UNGETCH(c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
268 return -2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
269 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
270 return c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
271 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
272
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
273
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
274 while(--max>=0){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
275 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
276
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
277 if(c==EOF)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
278 return EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
279
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
280 if(c=='\n'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
281 UNGETCH(c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
282 return -2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
283 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
284
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
285 if(base==16){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
286 if(!isxdigit(c))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
287 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
288
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
289 i<<=4;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
290
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
291 if(isdigit(c))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
292 i+=c-'0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
293 else if(i>='a')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
294 i+=0xa+c-'a';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
295 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
296 i+=0xa+c-'a';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
297
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
298 }else if(base==10){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
299 if(!isdigit(c))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
300 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
301 i*=10;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
302 i+=c-'0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
303 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
304 if(c<'0' || c>'7')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
305 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
306 i<<=3;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
307 i+=c-'0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
308 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
309 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
310
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
311 if(max>=0)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
312 UNGETCH(c);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
313
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
314 return i;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
315 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
316
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
317
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
318 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
319
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
320
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
321 static int scan_string(Token *tok, Tokenizer *tokz, bool escapes)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
322 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
323 STRING_DECL(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
324 int c;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
325
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
326 while(1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
327 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
328
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
329 if(c=='"')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
330 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
331
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
332 if(c=='\n'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
333 UNGETCH(c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
334 STRING_FREE(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
335 return E_TOKZ_UNEXPECTED_EOL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
336 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
337
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
338 if(c=='\\' && escapes){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
339 c=scan_char_escape(tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
340 if(c==-2){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
341 STRING_FREE(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
342 return E_TOKZ_UNEXPECTED_EOL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
343 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
344 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
345
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
346 if(c==EOF){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
347 STRING_FREE(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
348 return E_TOKZ_UNEXPECTED_EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
349 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
350
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
351 STRING_APPEND(s, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
352 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
353
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
354 STRING_FINISH(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
355
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
356 TOK_SET_STRING(tok, s);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
357
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
358 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
359 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
360
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
361
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
362 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
363
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
364
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
365 static int scan_char(Token *tok, Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
366 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
367 int c, c2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
368
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
369 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
370
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
371 if(c==EOF)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
372 return E_TOKZ_UNEXPECTED_EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
373
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
374 if(c=='\n')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
375 return E_TOKZ_UNEXPECTED_EOL;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
376
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
377 if(c=='\\'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
378 c=scan_char_escape(tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
379
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
380 if(c==EOF)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
381 return E_TOKZ_UNEXPECTED_EOF;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
382
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
383 if(c==-2)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
384 return E_TOKZ_UNEXPECTED_EOL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
385 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
386
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
387 c2=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
388
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
389 if(c2!='\'')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
390 return E_TOKZ_MULTICHAR;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
391
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
392 TOK_SET_CHAR(tok, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
393
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
394 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
395 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
396
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
397
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
398 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
399
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
400
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
401 #define START_IDENT(X) (isalpha(X) || X=='_' || X=='$')
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
402
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
403
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
404 static int scan_identifier(Token *tok, Tokenizer *tokz, int c)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
405 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
406 STRING_DECL(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
407
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
408 do{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
409 STRING_APPEND(s, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
410 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
411 }while(isalnum(c) || c=='_' || c=='$');
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
412
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
413 UNGETCH(c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
414
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
415 STRING_FINISH(s);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
416
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
417 TOK_SET_IDENT(tok, s);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
418
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
419 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
420 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
421
3
b1fbfab67908 trunk: changeset 6
tuomov
parents: 2
diff changeset
422 #define NP_SIMPLE_IMPL
60
a4033700e35c trunk: changeset 1313
tuomov
parents: 53
diff changeset
423 #include "np/numparser2.h"
a4033700e35c trunk: changeset 1313
tuomov
parents: 53
diff changeset
424 #include "np/np-conv.h"
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
425
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
426
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
427 static int scan_number(Token *tok, Tokenizer *tokz, int c)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
428 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
429 NPNum num=NUM_INIT;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
430 int e;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
431
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
432 if((e=parse_number(&num, tokz, c)))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
433 return e;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
434
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
435 if(num.type==NPNUM_INT){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
436 long l;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
437 if((e=num_to_long(&l, &num, TRUE)))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
438 return e;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
439
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
440 TOK_SET_LONG(tok, l);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
441 }else if(num.type==NPNUM_FLOAT){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
442 double d;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
443 if((e=num_to_double(&d, &num)))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
444 return e;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
445
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
446 TOK_SET_DOUBLE(tok, d);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
447 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
448 return E_TOKZ_NUMFMT;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
449 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
450
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
451 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
452 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
453
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
454
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
455 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
456
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
457
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
458 static uchar op_map[]={
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
459 0x00, /* ________ 0-7 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
460 0x00, /* ________ 8-15 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
461 0x00, /* ________ 16-23 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
462 0x00, /* ________ 24-31 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
463 0x62, /* _!___%&_ 32-39 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
464 0xff, /* ()*+,-./ 40-47 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
465 0x00, /* ________ 48-55 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
466 0xfc, /* __:;<=>? 56-63 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
467 0x01, /* @_______ 64-71 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
468 0x00, /* ________ 72-79 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
469 0x00, /* ________ 80-87 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
470 0x78, /* ___[_]^_ 88-95 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
471 0x00, /* ________ 96-103 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
472 0x00, /* ________ 104-111 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
473 0x00, /* ________ 112-119 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
474 0x38 /* ___{|}__ 120-127 */
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
475 };
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
476
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
477
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
478 static bool map_isset(uchar *map, uint ch)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
479 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
480 if(ch>127)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
481 return FALSE;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
482
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
483 return map[ch>>3]&(1<<(ch&7));
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
484 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
485
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
486
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
487 static bool is_opch(uint ch)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
488 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
489 return map_isset(op_map, ch);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
490 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
491
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
492
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
493 static int scan_op(Token *tok, Tokenizer *tokz, int c)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
494 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
495 int c2;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
496 int op=-1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
497
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
498 /* Quickly check it is an operator character */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
499 if(!is_opch(c))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
500 return E_TOKZ_INVALID_CHAR;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
501
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
502 switch(c){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
503 case '+':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
504 case '-':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
505 case '*':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
506 /* case '/': Checked elsewhere */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
507 case '%':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
508 case '^':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
509 case '!':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
510 case '=':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
511 case '<':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
512 case '>':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
513 c2=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
514 if(c2=='='){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
515 op=c|(c2<<8);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
516 }else if(c2==c && (c2!='%' && c2!='!' && c2!='*')){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
517 if(c=='<' || c=='>'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
518 int c3=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
519 if(c3=='='){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
520 op=c|(c2<<8)|(c3<<16);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
521 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
522 UNGETCH(c3);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
523 op=c|(c2<<8);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
524 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
525 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
526 op=c|(c2<<8);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
527 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
528 }else{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
529 UNGETCH(c2);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
530 op=c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
531 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
532 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
533
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
534 /* It is already known that it is a operator so these are not needed
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
535 case ':':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
536 case '~':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
537 case '?':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
538 case '.':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
539 case ';';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
540 case '{':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
541 case '}':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
542 case '@':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
543 case '|':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
544 case '&':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
545 */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
546 default:
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
547 op=c;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
548 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
549
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
550 TOK_SET_OP(tok, op);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
551
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
552 return 0;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
553 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
554
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
555
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
556 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
557
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
558
21
6d4282804e73 trunk: changeset 24
tuomov
parents: 20
diff changeset
559 void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...)
6d4282804e73 trunk: changeset 24
tuomov
parents: 20
diff changeset
560 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
561 va_list args;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
562
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
563 va_start(args, fmt);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
564
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
565 if(tokz!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
566 warn_obj_line_v(tokz->name, line, fmt, args);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
567 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
568 warn(fmt, args);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
569
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
570 va_end(args);
21
6d4282804e73 trunk: changeset 24
tuomov
parents: 20
diff changeset
571 }
6d4282804e73 trunk: changeset 24
tuomov
parents: 20
diff changeset
572
6d4282804e73 trunk: changeset 24
tuomov
parents: 20
diff changeset
573
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
574 void tokz_warn_error(const Tokenizer *tokz, int line, int e)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
575 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
576 if(e==E_TOKZ_UNEXPECTED_EOF)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
577 line=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
578
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
579 if(e<0)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
580 tokz_warn(tokz, line, "%s", strerror(-e));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
581 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
582 tokz_warn(tokz, line, "%s", TR(errors[e]));
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
583 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
584
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
585
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
586 bool tokz_get_token(Tokenizer *tokz, Token *tok)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
587 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
588 int c, c2, e;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
589
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
590 if (!(tokz->flags&TOKZ_READ_FROM_BUFFER))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
591 assert(tokz->file!=NULL);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
592
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
593 tok_free(tok);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
594
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
595 if(!TOK_IS_INVALID(&(tokz->ungettok))){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
596 *tok=tokz->ungettok;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
597 tokz->ungettok.type=TOK_INVALID;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
598 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
599 }
2
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
600
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
601 while(1){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
602
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
603 e=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
604
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
605 do{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
606 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
607 }while(c!='\n' && c!=EOF && isspace(c));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
608
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
609 tok->line=tokz->line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
610
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
611 switch(c){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
612 case EOF:
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
613 TOK_SET_OP(tok, OP_EOF);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
614 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
615
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
616 case '\n':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
617 INC_LINE();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
618
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
619 if(tokz->flags&TOKZ_IGNORE_NEXTLINE)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
620 continue;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
621
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
622 TOK_SET_OP(tok, OP_NEXTLINE);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
623
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
624 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
625
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
626 case '\\':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
627 do{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
628 c=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
629 if(c==EOF){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
630 TOK_SET_OP(tok, OP_EOF);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
631 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
632 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
633 if(!isspace(c) && e==0){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
634 e=E_TOKZ_EOL_EXPECTED;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
635 tokz_warn_error(tokz, tokz->line, e);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
636 if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
637 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
638 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
639 }while(c!='\n');
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
640
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
641 INC_LINE();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
642 continue;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
643
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
644 case '#':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
645 if(tokz->flags&TOKZ_READ_COMMENTS){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
646 e=scan_line_comment(tok, tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
647 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
648 }else if((e=skip_line_comment(tokz))){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
649 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
650 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
651
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
652 continue;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
653
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
654 case '/':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
655 c2=GETCH();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
656
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
657 if(c2=='='){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
658 TOK_SET_OP(tok, OP_AS_DIV);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
659 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
660 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
661
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
662 if(c2!='*'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
663 UNGETCH(c2);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
664 TOK_SET_OP(tok, OP_DIV);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
665 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
666 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
667
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
668 if(tokz->flags&TOKZ_READ_COMMENTS){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
669 e=scan_c_comment(tok, tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
670 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
671 }else if((e=skip_c_comment(tokz))){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
672 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
673 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
674
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
675 continue;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
676
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
677 case '\"':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
678 e=scan_string(tok, tokz, TRUE);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
679 break;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
680
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
681 case '\'':
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
682 e=scan_char(tok, tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
683 break;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
684
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
685 default:
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
686 if(('0'<=c && c<='9') || c=='-' || c=='+'){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
687 e=scan_number(tok, tokz, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
688 break;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
689 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
690
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
691 if(START_IDENT(c))
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
692 e=scan_identifier(tok, tokz, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
693 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
694 e=scan_op(tok, tokz, c);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
695 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
696
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
697 if(!e)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
698 return TRUE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
699
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
700 tokz_warn_error(tokz, tokz->line, e);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
701 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
702 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
703 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
704
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
705
2
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
706 void tokz_unget_token(Tokenizer *tokz, Token *tok)
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
707 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
708 tok_free(&(tokz->ungettok));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
709 tokz->ungettok=*tok;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
710 tok->type=TOK_INVALID;
2
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
711 }
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
712
e14a1aba4c56 trunk: changeset 5
tuomov
parents: 1
diff changeset
713
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
714 /*
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
715 * File open
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
716 */
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
717
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
718 static bool do_tokz_pushf(Tokenizer *tokz)
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
719 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
720 Tokenizer_FInfo *finfo;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
721
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
722 finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
723 tokz->filestack_n, tokz->filestack_n+1);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
724
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
725 if(finfo==NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
726 return FALSE;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
727
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
728 tokz->filestack=finfo;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
729 finfo=&(finfo[tokz->filestack_n++]);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
730
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
731 finfo->file=tokz->file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
732 finfo->name=tokz->name;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
733 finfo->line=tokz->line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
734 finfo->ungetc=tokz->ungetc;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
735 finfo->ungettok=tokz->ungettok;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
736
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
737 return TRUE;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
738 }
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
739
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
740
14
91f0c87b62a7 trunk: changeset 17
tuomov
parents: 9
diff changeset
741 bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname)
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
742 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
743 char *fname_copy=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
744
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
745 if(file==NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
746 return FALSE;
14
91f0c87b62a7 trunk: changeset 17
tuomov
parents: 9
diff changeset
747
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
748 if(fname!=NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
749 fname_copy=scopy(fname);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
750 if(fname_copy==NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
751 warn_err();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
752 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
753 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
754 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
755
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
756 if(tokz->file!=NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
757 if(!do_tokz_pushf(tokz)){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
758 warn_err();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
759 if(fname_copy!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
760 free(fname_copy);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
761 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
762 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
763 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
764
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
765 tokz->file=file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
766 tokz->name=fname_copy;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
767 tokz->line=1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
768 tokz->ungetc=-1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
769 tokz->ungettok.type=TOK_INVALID;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
770
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
771 return TRUE;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
772 }
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
773
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
774
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
775 bool tokz_pushf(Tokenizer *tokz, const char *fname)
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
776 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
777 FILE *file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
778
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
779 file=fopen(fname, "r");
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
780
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
781 if(file==NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
782 warn_err_obj(fname);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
783 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
784 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
785
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
786 if(!tokz_pushf_file(tokz, file, fname)){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
787 fclose(file);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
788 return FALSE;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
789 }
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
790
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
791 return TRUE;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
792 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
793
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
794
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
795
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
796 static Tokenizer *tokz_create()
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
797 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
798 Tokenizer*tokz;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
799
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
800 tokz=ALLOC(Tokenizer);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
801
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
802 if(tokz==NULL){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
803 warn_err();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
804 return NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
805 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
806
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
807 tokz->file=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
808 tokz->name=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
809 tokz->line=1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
810 tokz->ungetc=-1;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
811 tokz->ungettok.type=TOK_INVALID;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
812 tokz->flags=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
813 tokz->optstack=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
814 tokz->nest_lvl=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
815 tokz->filestack_n=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
816 tokz->filestack=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
817 tokz->buffer.data=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
818 tokz->buffer.len=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
819 tokz->buffer.pos=0;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
820
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
821 return tokz;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
822 }
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
823
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
824
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
825 Tokenizer *tokz_open(const char *fname)
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
826 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
827 Tokenizer *tokz;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
828
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
829 tokz=tokz_create();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
830
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
831 if(!tokz_pushf(tokz, fname)){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
832 free(tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
833 return NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
834 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
835
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
836 return tokz;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
837 }
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
838
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
839
14
91f0c87b62a7 trunk: changeset 17
tuomov
parents: 9
diff changeset
840 Tokenizer *tokz_open_file(FILE *file, const char *fname)
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
841 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
842 Tokenizer *tokz;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
843
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
844 tokz=tokz_create();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
845
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
846 if(!tokz_pushf_file(tokz, file, fname)){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
847 free(tokz);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
848 return NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
849 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
850
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
851 return tokz;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
852 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
853
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
854 Tokenizer *tokz_prepare_buffer(char *buffer, int len)
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
855 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
856 Tokenizer *tokz;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
857 char old=0;
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
858
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
859 tokz=tokz_create();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
860 if(len>0){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
861 old=buffer[len-1];
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
862 buffer[len-1]='\0';
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
863 }
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
864
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
865 tokz->flags|=TOKZ_READ_FROM_BUFFER;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
866 tokz->buffer.data=scopy(buffer);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
867 tokz->buffer.len=(len>0 ? (uint)len : strlen(tokz->buffer.data));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
868 tokz->buffer.pos=0;
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
869
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
870 if(old>0)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
871 buffer[len-1]=old;
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
872
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
873 return tokz;
35
5a71d53d0228 trunk: changeset 38
tuomov
parents: 21
diff changeset
874 }
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
875
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
876 /*
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
877 * File close
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
878 */
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
879
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
880 static bool do_tokz_popf(Tokenizer *tokz, bool shrink)
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
881 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
882 Tokenizer_FInfo *finfo;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
883
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
884 if(tokz->filestack_n<=0)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
885 return FALSE;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
886
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
887 if(tokz->file!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
888 fclose(tokz->file);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
889 if(tokz->name!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
890 free(tokz->name);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
891
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
892 finfo=&(tokz->filestack[--tokz->filestack_n]);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
893
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
894 tokz->file=finfo->file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
895 tokz->name=finfo->name;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
896 tokz->line=finfo->line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
897 tokz->ungetc=finfo->ungetc;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
898 tokz->ungettok=finfo->ungettok;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
899
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
900 if(tokz->filestack_n==0){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
901 free(tokz->filestack);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
902 tokz->filestack=NULL;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
903 }else if(shrink){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
904 finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
905 tokz->filestack_n+1, tokz->filestack_n);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
906 if(finfo==NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
907 warn_err();
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
908 else
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
909 tokz->filestack=finfo;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
910 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
911
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
912 return TRUE;
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
913 }
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
914
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
915
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
916 bool tokz_popf(Tokenizer *tokz)
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
917 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
918 return do_tokz_popf(tokz, TRUE);
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
919 }
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
920
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
921
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
922 void tokz_close(Tokenizer *tokz)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
923 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
924 while(tokz->filestack_n>0)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
925 do_tokz_popf(tokz, FALSE);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
926
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
927 if(tokz->file!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
928 fclose(tokz->file);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
929 if(tokz->name!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
930 free(tokz->name);
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
931 tok_free(&(tokz->ungettok));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
932
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
933 free(tokz);
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
934 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
935
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
936
1
6e704fc09528 trunk: changeset 4
tuomov
parents: 0
diff changeset
937
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
938 /* */
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
939
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
940
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
941 void tok_free(Token *tok)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
942 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
943 if(TOK_IS_STRING(tok) || TOK_IS_IDENT(tok) || TOK_IS_COMMENT(tok)){
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
944 if(TOK_STRING_VAL(tok)!=NULL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
945 free(TOK_STRING_VAL(tok));
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
946 }
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
947
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
948 tok->type=TOK_INVALID;
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
949 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
950
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
951
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
952 void tok_init(Token *tok)
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
953 {
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
954 static Token dummy=TOK_INIT;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
955
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 60
diff changeset
956 memcpy(tok, &dummy, sizeof(*tok));
0
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
957 }
86b7f6f9c5c0 Tailorization of trunk
tailor@f281.ttorni.ton.tut.fi
parents:
diff changeset
958

mercurial