Mercurial > repos > libtu-3 / file comparison

comparison: include/libtu/tokenizer.h

include/libtu/tokenizer.h

changeset 5
f878a9ffa3e0
child 8
c1994196683f
equal deleted inserted replaced
4:ee28b655297b 5:f878a9ffa3e0
1 /*
2 * libtu/tokenizer.h
3 *
4 * Copyright (c) Tuomo Valkonen 1999-2000.
5 *
6 * This file is distributed under the terms of the "Artistic License".
7 * See the included file LICENSE for details.
8 */
9
10 #ifndef __LIBTU_TOKENIZER_H
11 #define __LIBTU_TOKENIZER_H
12
13 #include <stdio.h>
14 #include "types.h"
15
16
17 #define TOK_SET_LONG(TOK, VAL) {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
18 #define TOK_SET_DOUBLE(TOK, VAL) {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
19 #define TOK_SET_CHAR(TOK, VAL) {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
20 #define TOK_SET_STRING(TOK, VAL) {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
21 #define TOK_SET_IDENT(TOK, VAL) {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
22 #define TOK_SET_COMMENT(TOK, VAL) {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
23 #define TOK_SET_OP(TOK, VAL) {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}
24
25 #define TOK_TYPE(TOK) ((TOK)->type)
26 #define TOK_LONG_VAL(TOK) ((TOK)->u.lval)
27 #define TOK_DOUBLE_VAL(TOK) ((TOK)->u.dval)
28 #define TOK_CHAR_VAL(TOK) ((TOK)->u.cval)
29 #define TOK_STRING_VAL(TOK) ((TOK)->u.sval)
30 #define TOK_IDENT_VAL(TOK) ((TOK)->u.sval)
31 #define TOK_COMMENT_VAL(TOK) ((TOK)->u.sval)
32 #define TOK_OP_VAL(TOK) ((TOK)->u.opval)
33
34 #define TOK_IS_INVALID(TOK) ((TOK)->type==TOK_INVALID)
35 #define TOK_IS_LONG(TOK) ((TOK)->type==TOK_LONG)
36 #define TOK_IS_DOUBLE(TOK) ((TOK)->type==TOK_DOUBLE)
37 #define TOK_IS_CHAR(TOK) ((TOK)->type==TOK_CHAR)
38 #define TOK_IS_STRING(TOK) ((TOK)->type==TOK_STRING)
39 #define TOK_IS_IDENT(TOK) ((TOK)->type==TOK_IDENT)
40 #define TOK_IS_COMMENT(TOK) ((TOK)->type==TOK_COMMENT)
41 #define TOK_IS_OP(TOK) ((TOK)->type==TOK_OP)
42
43 #define TOK_OP_IS(TOK, OP) ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))
44
45 #define TOK_TAKE_STRING_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
46 #define TOK_TAKE_IDENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
47 #define TOK_TAKE_COMMENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
48
49
50 enum{
51 TOK_INVALID=0,
52 TOK_LONG,
53 TOK_DOUBLE,
54 TOK_CHAR,
55 TOK_STRING,
56 TOK_IDENT,
57 TOK_COMMENT,
58 TOK_OP
59 };
60
61
62 enum{
63 #define OP2(X,Y) ((X)|((Y)<<8))
64 #define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))
65
66 OP_L_PAR= '(', OP_R_PAR= ')', OP_L_BRK= '[', OP_R_BRK= ']',
67 OP_L_BRC= '{', OP_R_BRC= '}', OP_COMMA= ',', OP_SCOLON= ';',
68
69 OP_PLUS= '+', OP_MINUS= '-', OP_MUL= '*', OP_DIV= '/',
70 OP_MOD= '%', OP_POW= '^', OP_OR= '|', OP_AND= '&',
71 /*OP_NOT= '~',*/ OP_NOT= '!', OP_ASGN= '=', OP_LT= '<',
72 OP_GT= '>', OP_DOT= '.', OP_COLON= ':', OP_QMARK= '?',
73 OP_AT= '@',
74 OP_NEXTLINE='\n',OP_EOF= -1,
75
76 OP_INC= OP2('+','+'), OP_DEC= OP2('-','-'),
77 OP_LSHIFT= OP2('<','<'), OP_RSHIFT= OP2('>','>'),
78 OP_AS_INC= OP2('+','='), OP_AS_DEC= OP2('-','='),
79 OP_AS_MUL= OP2('*','='), OP_AS_DIV= OP2('/','='),
80 OP_AS_MOD= OP2('%','='), OP_AS_POW= OP2('^','='),
81
82 /* AS_OR= OP2('|','='), AS_AND= OP2('&','='), */
83 OP_EQ= OP2('=','='), OP_NE= OP2('!','='),
84 OP_LE= OP2('<','='), OP_GE= OP2('>','=')
85
86 /* L_AND= OP2('&','&'), L_OR= OP2('|','|'),
87 L_XOR= OP2('^','^'), */
88
89 /* AsLShift= OP3('<','<','='),
90 AsRShift= OP3('>','>','='), */
91
92 #undef OP2
93 #undef OP3
94 };
95
96
97 typedef struct{
98 int type;
99 int line;
100 union{
101 long lval;
102 double dval;
103 char cval;
104 char *sval;
105 int opval;
106 } u;
107 } Token;
108
109 #define TOK_INIT {0, 0, {0}}
110
111
112 extern void tok_free(Token*tok);
113 extern void tok_init(Token*tok);
114
115
116 /* */
117
118
119 enum{
120 TOKZ_IGNORE_NEXTLINE=0x1,
121 TOKZ_READ_COMMENTS=0x2,
122 TOKZ_PARSER_INDENT_MODE=0x04,
123 TOKZ_ERROR_TOLERANT=0x8
124 };
125
126
127 enum{
128 E_TOKZ_UNEXPECTED_EOF=1,
129 E_TOKZ_UNEXPECTED_EOL,
130 E_TOKZ_EOL_EXPECTED,
131 E_TOKZ_INVALID_CHAR,
132 E_TOKZ_TOOBIG,
133 E_TOKZ_NUMFMT,
134 E_TOKZ_NUM_JUNK,
135 E_TOKZ_NOTINT,
136 E_TOKZ_RANGE,
137 E_TOKZ_MULTICHAR,
138
139 E_TOKZ_TOKEN_LIMIT,
140 E_TOKZ_UNKNOWN_OPTION,
141 E_TOKZ_SYNTAX,
142 E_TOKZ_INVALID_ARGUMENT,
143 E_TOKZ_EOS_EXPECTED,
144 E_TOKZ_TOO_FEW_ARGS,
145 E_TOKZ_TOO_MANY_ARGS,
146 E_TOKZ_MAX_NEST,
147 E_TOKZ_IDENTIFIER_EXPECTED,
148
149 E_TOKZ_LBRACE_EXPECTED
150 };
151
152
153 struct _ConfOpt;
154
155 typedef struct _Tokenizer_FInfo{
156 FILE *file;
157 char *name;
158 int line;
159 int ungetc;
160 Token ungettok;
161 } Tokenizer_FInfo;
162
163 typedef struct _Tokenizer{
164 FILE *file;
165 char *name;
166 int line;
167 int ungetc;
168 Token ungettok;
169
170 int flags;
171 const struct _ConfOpt **optstack;
172 int nest_lvl;
173 void *user_data;
174
175 int filestack_n;
176 Tokenizer_FInfo *filestack;
177 } Tokenizer;
178
179
180 extern Tokenizer *tokz_open(const char *fname);
181 extern Tokenizer *tokz_open_file(FILE *file);
182 extern void tokz_close(Tokenizer *tokz);
183 extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
184 extern void tokz_unget_token(Tokenizer *tokz, Token *tok);
185 extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
186
187 extern bool tokz_pushf(Tokenizer *tokz, const char *fname);
188 extern bool tokz_pushf_file(Tokenizer *tokz, FILE *file);
189 extern bool tokz_popf(Tokenizer *tokz);
190
191 #endif /* __LIBTU_TOKENIZER_H */

Mercurial Repository: libtu-3

mercurial