--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/include/tokenizer.h Tue Feb 15 18:57:52 2005 +0100
@@ -0,0 +1,170 @@
+/*
+ * libtu/tokenizer.h
+ *
+ * Copyright (c) Tuomo Valkonen 1999-2000.
+ *
+ * This file is distributed under the terms of the "Artistic License".
+ * See the included file LICENSE for details.
+ */
+
+#ifndef __LIBTU_TOKENIZER_H
+#define __LIBTU_TOKENIZER_H
+
+#include <stdio.h>
+#include "types.h"
+
+
+#define TOK_SET_LONG(TOK, VAL) {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
+#define TOK_SET_DOUBLE(TOK, VAL) {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
+#define TOK_SET_CHAR(TOK, VAL) {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
+#define TOK_SET_STRING(TOK, VAL) {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
+#define TOK_SET_IDENT(TOK, VAL) {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
+#define TOK_SET_COMMENT(TOK, VAL) {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
+#define TOK_SET_OP(TOK, VAL) {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}
+
+#define TOK_TYPE(TOK) ((TOK)->type)
+#define TOK_LONG_VAL(TOK) ((TOK)->u.lval)
+#define TOK_DOUBLE_VAL(TOK) ((TOK)->u.dval)
+#define TOK_CHAR_VAL(TOK) ((TOK)->u.cval)
+#define TOK_STRING_VAL(TOK) ((TOK)->u.sval)
+#define TOK_IDENT_VAL(TOK) ((TOK)->u.sval)
+#define TOK_COMMENT_VAL(TOK) ((TOK)->u.sval)
+#define TOK_OP_VAL(TOK) ((TOK)->u.opval)
+
+#define TOK_IS_LONG(TOK) ((TOK)->type==TOK_LONG)
+#define TOK_IS_DOUBLE(TOK) ((TOK)->type==TOK_DOUBLE)
+#define TOK_IS_CHAR(TOK) ((TOK)->type==TOK_CHAR)
+#define TOK_IS_STRING(TOK) ((TOK)->type==TOK_STRING)
+#define TOK_IS_IDENT(TOK) ((TOK)->type==TOK_IDENT)
+#define TOK_IS_COMMENT(TOK) ((TOK)->type==TOK_COMMENT)
+#define TOK_IS_OP(TOK) ((TOK)->type==TOK_OP)
+
+#define TOK_OP_IS(TOK, OP) ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))
+
+#define TOK_TAKE_STRING_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
+#define TOK_TAKE_IDENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
+#define TOK_TAKE_COMMENT_VAL(TOK) ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
+
+
+enum{
+ TOK_INVALID=0,
+ TOK_LONG,
+ TOK_DOUBLE,
+ TOK_CHAR,
+ TOK_STRING,
+ TOK_IDENT,
+ TOK_COMMENT,
+ TOK_OP
+};
+
+
+enum{
+#define OP2(X,Y) ((X)|((Y)<<8))
+#define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))
+
+ OP_L_PAR= '(', OP_R_PAR= ')', OP_L_BRK= '[', OP_R_BRK= ']',
+ OP_L_BRC= '{', OP_R_BRC= '}', OP_COMMA= ',', OP_SCOLON= ';',
+
+ OP_PLUS= '+', OP_MINUS= '-', OP_MUL= '*', OP_DIV= '/',
+ OP_MOD= '%', OP_POW= '^', OP_OR= '|', OP_AND= '&',
+ /*OP_NOT= '~',*/ OP_NOT= '!', OP_ASGN= '=', OP_LT= '<',
+ OP_GT= '>', OP_DOT= '.', OP_COLON= ':', OP_QMARK= '?',
+ OP_AT= '@',
+ OP_NEXTLINE='\n',OP_EOF= -1,
+
+ OP_INC= OP2('+','+'), OP_DEC= OP2('-','-'),
+ OP_LSHIFT= OP2('<','<'), OP_RSHIFT= OP2('>','>'),
+ OP_AS_INC= OP2('+','='), OP_AS_DEC= OP2('-','='),
+ OP_AS_MUL= OP2('*','='), OP_AS_DIV= OP2('/','='),
+ OP_AS_MOD= OP2('%','='), OP_AS_POW= OP2('^','='),
+
+/* AS_OR= OP2('|','='), AS_AND= OP2('&','='), */
+ OP_EQ= OP2('=','='), OP_NE= OP2('!','='),
+ OP_LE= OP2('<','='), OP_GE= OP2('>','=')
+
+/* L_AND= OP2('&','&'), L_OR= OP2('|','|'),
+ L_XOR= OP2('^','^'), */
+
+/* AsLShift= OP3('<','<','='),
+ AsRShift= OP3('>','>','='), */
+
+#undef OP2
+#undef OP3
+};
+
+
+typedef struct{
+ int type;
+ int line;
+ union{
+ long lval;
+ double dval;
+ char cval;
+ char *sval;
+ int opval;
+ } u;
+} Token;
+
+#define TOK_INIT {0, 0, {0}}
+
+
+extern void tok_free(Token*tok);
+extern void tok_init(Token*tok);
+
+
+/* */
+
+
+enum{
+ TOKZ_IGNORE_NEXTLINE=0x1,
+ TOKZ_READ_COMMENTS=0x2,
+ TOKZ_PARSER_INDENT_MODE=0x4
+};
+
+
+enum{
+ E_TOKZ_UNEXPECTED_EOF=1,
+ E_TOKZ_UNEXPECTED_EOL,
+ E_TOKZ_EOL_EXPECTED,
+ E_TOKZ_INVALID_CHAR,
+ E_TOKZ_TOOBIG,
+ E_TOKZ_NUMFMT,
+ E_TOKZ_NUM_JUNK,
+ E_TOKZ_NOTINT,
+ E_TOKZ_RANGE,
+ E_TOKZ_MULTICHAR,
+
+ E_TOKZ_TOKEN_LIMIT,
+ E_TOKZ_UNKNOWN_OPTION,
+ E_TOKZ_SYNTAX,
+ E_TOKZ_INVALID_ARGUMENT,
+ E_TOKZ_EOS_EXPECTED,
+ E_TOKZ_TOO_FEW_ARGS,
+ E_TOKZ_TOO_MANY_ARGS,
+ E_TOKZ_MAX_NEST,
+ E_TOKZ_UNEXPECTED_EOS,
+ E_TOKZ_IDENTIFIER_EXPECTED
+};
+
+
+struct _ConfOpt;
+
+typedef struct{
+ FILE *file;
+ const char *name;
+ int flags;
+ int line;
+ int ungetc;
+ const struct _ConfOpt **optstack;
+ int nest_lvl;
+ void *user_data;
+} Tokenizer;
+
+
+extern Tokenizer *tokz_open(const char *fname);
+extern Tokenizer *tokz_open_file(FILE *file);
+extern void tokz_close(Tokenizer *tokz);
+extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
+extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
+
+#endif /* __LIBTU_TOKENIZER_H */