tokenizer.h

Mon, 20 Apr 2020 10:14:32 -0500

author
Tuomo Valkonen <tuomov@iki.fi>
date
Mon, 20 Apr 2020 10:14:32 -0500
changeset 119
87e3bb5086e8
parent 62
aae5facf9fc5
permissions
-rw-r--r--

Convert README to markdown

58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
1 /*
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
2 * libtu/tokenizer.h
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
3 *
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
4 * Copyright (c) Tuomo Valkonen 1999-2002.
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
5 *
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
6 * You may distribute and modify this library under the terms of either
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
7 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
8 */
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
9
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
10 #ifndef LIBTU_TOKENIZER_H
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
11 #define LIBTU_TOKENIZER_H
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
12
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
13 #include <stdio.h>
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
14 #include "types.h"
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
15
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
16
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
17 #define TOK_SET_BOOL(TOK, VAL) {(TOK)->type=TOK_BOOL; (TOK)->u.bval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
18 #define TOK_SET_LONG(TOK, VAL) {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
19 #define TOK_SET_DOUBLE(TOK, VAL) {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
20 #define TOK_SET_CHAR(TOK, VAL) {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
21 #define TOK_SET_STRING(TOK, VAL) {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
22 #define TOK_SET_IDENT(TOK, VAL) {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
23 #define TOK_SET_COMMENT(TOK, VAL) {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
24 #define TOK_SET_OP(TOK, VAL) {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
25
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
26 #define TOK_TYPE(TOK) ((TOK)->type)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
27 #define TOK_BOOL_VAL(TOK) ((TOK)->u.bval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
28 #define TOK_LONG_VAL(TOK) ((TOK)->u.lval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
29 #define TOK_DOUBLE_VAL(TOK) ((TOK)->u.dval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
30 #define TOK_CHAR_VAL(TOK) ((TOK)->u.cval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
31 #define TOK_STRING_VAL(TOK) ((TOK)->u.sval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
32 #define TOK_IDENT_VAL(TOK) ((TOK)->u.sval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
33 #define TOK_COMMENT_VAL(TOK) ((TOK)->u.sval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
34 #define TOK_OP_VAL(TOK) ((TOK)->u.opval)
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
35
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
36 #define TOK_IS_INVALID(TOK) ((TOK)->type==TOK_INVALID)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
37 #define TOK_IS_BOOL(TOK) ((TOK)->type==TOK_BOOL)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
38 #define TOK_IS_LONG(TOK) ((TOK)->type==TOK_LONG)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
39 #define TOK_IS_DOUBLE(TOK) ((TOK)->type==TOK_DOUBLE)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
40 #define TOK_IS_CHAR(TOK) ((TOK)->type==TOK_CHAR)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
41 #define TOK_IS_STRING(TOK) ((TOK)->type==TOK_STRING)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
42 #define TOK_IS_IDENT(TOK) ((TOK)->type==TOK_IDENT)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
43 #define TOK_IS_COMMENT(TOK) ((TOK)->type==TOK_COMMENT)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
44 #define TOK_IS_OP(TOK) ((TOK)->type==TOK_OP)
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
45
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
46 #define TOK_OP_IS(TOK, OP) ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
47
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
48 #define TOK_TAKE_STRING_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
49 #define TOK_TAKE_IDENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
50 #define TOK_TAKE_COMMENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
51
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
52
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
53 enum{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
54 TOK_INVALID=0,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
55 TOK_LONG='l',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
56 TOK_DOUBLE='d',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
57 TOK_CHAR='c',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
58 TOK_STRING='s',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
59 TOK_IDENT='i',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
60 TOK_BOOL='b',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
61 TOK_COMMENT='#',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
62 TOK_OP='+'
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
63 };
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
64
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
65
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
66 enum{
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
67 #define OP2(X,Y) ((X)|((Y)<<8))
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
68 #define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
69
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
70 OP_L_PAR= '(', OP_R_PAR= ')', OP_L_BRK= '[', OP_R_BRK= ']',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
71 OP_L_BRC= '{', OP_R_BRC= '}', OP_COMMA= ',', OP_SCOLON= ';',
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
72
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
73 OP_PLUS= '+', OP_MINUS= '-', OP_MUL= '*', OP_DIV= '/',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
74 OP_MOD= '%', OP_POW= '^', OP_OR= '|', OP_AND= '&',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
75 /*OP_NOT= '~',*/ OP_NOT= '!', OP_ASGN= '=', OP_LT= '<',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
76 OP_GT= '>', OP_DOT= '.', OP_COLON= ':', OP_QMARK= '?',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
77 OP_AT= '@',
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
78 OP_NEXTLINE='\n',OP_EOF= -1,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
79
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
80 OP_INC= OP2('+','+'), OP_DEC= OP2('-','-'),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
81 OP_LSHIFT= OP2('<','<'), OP_RSHIFT= OP2('>','>'),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
82 OP_AS_INC= OP2('+','='), OP_AS_DEC= OP2('-','='),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
83 OP_AS_MUL= OP2('*','='), OP_AS_DIV= OP2('/','='),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
84 OP_AS_MOD= OP2('%','='), OP_AS_POW= OP2('^','='),
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
85
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
86 /* AS_OR= OP2('|','='), AS_AND= OP2('&','='), */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
87 OP_EQ= OP2('=','='), OP_NE= OP2('!','='),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
88 OP_LE= OP2('<','='), OP_GE= OP2('>','=')
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
89
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
90 /* L_AND= OP2('&','&'), L_OR= OP2('|','|'),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
91 L_XOR= OP2('^','^'), */
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
92
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
93 /* AsLShift= OP3('<','<','='),
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
94 AsRShift= OP3('>','>','='), */
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
95
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
96 #undef OP2
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
97 #undef OP3
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
98 };
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
99
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
100
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
101 typedef struct{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
102 int type;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
103 int line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
104 union{
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
105 bool bval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
106 long lval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
107 double dval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
108 char cval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
109 char *sval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
110 int opval;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
111 } u;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
112 } Token;
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
113
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
114 #define TOK_INIT {0, 0, {0}}
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
115
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
116
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
117 extern void tok_free(Token*tok);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
118 extern void tok_init(Token*tok);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
119
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
120
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
121 /* */
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
122
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
123
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
124 enum{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
125 TOKZ_IGNORE_NEXTLINE=0x1,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
126 TOKZ_READ_COMMENTS=0x2,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
127 TOKZ_PARSER_INDENT_MODE=0x04,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
128 TOKZ_ERROR_TOLERANT=0x8,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
129 TOKZ_READ_FROM_BUFFER=0x10,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
130 TOKZ_DEFAULT_OPTION=0x20
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
131 };
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
132
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
133
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
134 enum{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
135 E_TOKZ_UNEXPECTED_EOF=1,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
136 E_TOKZ_UNEXPECTED_EOL,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
137 E_TOKZ_EOL_EXPECTED,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
138 E_TOKZ_INVALID_CHAR,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
139 E_TOKZ_TOOBIG,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
140 E_TOKZ_NUMFMT,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
141 E_TOKZ_NUM_JUNK,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
142 E_TOKZ_NOTINT,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
143 E_TOKZ_RANGE,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
144 E_TOKZ_MULTICHAR,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
145
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
146 E_TOKZ_TOKEN_LIMIT,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
147 E_TOKZ_UNKNOWN_OPTION,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
148 E_TOKZ_SYNTAX,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
149 E_TOKZ_INVALID_ARGUMENT,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
150 E_TOKZ_EOS_EXPECTED,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
151 E_TOKZ_TOO_FEW_ARGS,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
152 E_TOKZ_TOO_MANY_ARGS,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
153 E_TOKZ_MAX_NEST,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
154 E_TOKZ_IDENTIFIER_EXPECTED,
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
155
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
156 E_TOKZ_LBRACE_EXPECTED
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
157 };
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
158
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
159
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
160 struct _ConfOpt;
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
161
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
162 typedef struct _Tokenizer_FInfo{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
163 FILE *file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
164 char *name;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
165 int line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
166 int ungetc;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
167 Token ungettok;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
168 } Tokenizer_FInfo;
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
169
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
170 typedef struct _Tokenizer_Buffer{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
171 char *data;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
172 int len;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
173 int pos;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
174 } Tokenizer_Buffer;
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
175
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
176 typedef struct _Tokenizer{
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
177 FILE *file;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
178 char *name;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
179 int line;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
180 int ungetc;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
181 Token ungettok;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
182
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
183 Tokenizer_Buffer buffer;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
184
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
185 int flags;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
186 const struct _ConfOpt **optstack;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
187 int nest_lvl;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
188 void *user_data;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
189
62
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
190 int filestack_n;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
191 Tokenizer_FInfo *filestack;
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
192
aae5facf9fc5 trunk: changeset 1318
tuomov
parents: 58
diff changeset
193 char **includepaths;
58
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
194 } Tokenizer;
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
195
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
196
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
197 extern Tokenizer *tokz_open(const char *fname);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
198 extern Tokenizer *tokz_open_file(FILE *file, const char *fname);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
199 extern Tokenizer *tokz_prepare_buffer(char *buffer, int len);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
200 extern void tokz_close(Tokenizer *tokz);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
201 extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
202 extern void tokz_unget_token(Tokenizer *tokz, Token *tok);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
203 extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
204 extern void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
205
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
206 extern bool tokz_pushf(Tokenizer *tokz, const char *fname);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
207 extern bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
208 extern bool tokz_popf(Tokenizer *tokz);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
209
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
210 extern void tokz_set_includepaths(Tokenizer *tokz, char **paths);
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
211
789df543d0c3 trunk: changeset 1303
tuomov
parents:
diff changeset
212 #endif /* LIBTU_TOKENIZER_H */

mercurial