tokenizer.h

/*
 * libtu/tokenizer.h
 *
 * Copyright (c) Tuomo Valkonen 1999-2002. 
 *
 * You may distribute and modify this library under the terms of either
 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
 */

#ifndef LIBTU_TOKENIZER_H
#define LIBTU_TOKENIZER_H

#include <stdio.h>
#include "types.h"


#define TOK_SET_BOOL(TOK, VAL)         {(TOK)->type=TOK_BOOL; (TOK)->u.bval=VAL;}
#define TOK_SET_LONG(TOK, VAL)         {(TOK)->type=TOK_LONG; (TOK)->u.lval=VAL;}
#define TOK_SET_DOUBLE(TOK, VAL)     {(TOK)->type=TOK_DOUBLE; (TOK)->u.dval=VAL;}
#define TOK_SET_CHAR(TOK, VAL)         {(TOK)->type=TOK_CHAR; (TOK)->u.cval=VAL;}
#define TOK_SET_STRING(TOK, VAL)     {(TOK)->type=TOK_STRING; (TOK)->u.sval=VAL;}
#define TOK_SET_IDENT(TOK, VAL)     {(TOK)->type=TOK_IDENT; (TOK)->u.sval=VAL;}
#define TOK_SET_COMMENT(TOK, VAL)     {(TOK)->type=TOK_COMMENT; (TOK)->u.sval=VAL;}
#define TOK_SET_OP(TOK, VAL)         {(TOK)->type=TOK_OP; (TOK)->u.opval=VAL;}

#define TOK_TYPE(TOK)                ((TOK)->type)
#define TOK_BOOL_VAL(TOK)            ((TOK)->u.bval)
#define TOK_LONG_VAL(TOK)            ((TOK)->u.lval)
#define TOK_DOUBLE_VAL(TOK)            ((TOK)->u.dval)
#define TOK_CHAR_VAL(TOK)            ((TOK)->u.cval)
#define TOK_STRING_VAL(TOK)            ((TOK)->u.sval)
#define TOK_IDENT_VAL(TOK)            ((TOK)->u.sval)
#define TOK_COMMENT_VAL(TOK)        ((TOK)->u.sval)
#define TOK_OP_VAL(TOK)                ((TOK)->u.opval)

#define TOK_IS_INVALID(TOK)            ((TOK)->type==TOK_INVALID)
#define TOK_IS_BOOL(TOK)            ((TOK)->type==TOK_BOOL)
#define TOK_IS_LONG(TOK)            ((TOK)->type==TOK_LONG)
#define TOK_IS_DOUBLE(TOK)            ((TOK)->type==TOK_DOUBLE)
#define TOK_IS_CHAR(TOK)            ((TOK)->type==TOK_CHAR)
#define TOK_IS_STRING(TOK)            ((TOK)->type==TOK_STRING)
#define TOK_IS_IDENT(TOK)            ((TOK)->type==TOK_IDENT)
#define TOK_IS_COMMENT(TOK)            ((TOK)->type==TOK_COMMENT)
#define TOK_IS_OP(TOK)                ((TOK)->type==TOK_OP)

#define TOK_OP_IS(TOK, OP)            ((TOK)->type==TOK_OP && (TOK)->u.opval==(OP))

#define TOK_TAKE_STRING_VAL(TOK)    ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
#define TOK_TAKE_IDENT_VAL(TOK)        ((TOK)->type=TOK_INVALID, (TOK)->u.sval)
#define TOK_TAKE_COMMENT_VAL(TOK)    ((TOK)->type=TOK_INVALID, (TOK)->u.sval)


enum{
    TOK_INVALID=0,
    TOK_LONG='l',
    TOK_DOUBLE='d',
    TOK_CHAR='c',
    TOK_STRING='s',
    TOK_IDENT='i',
    TOK_BOOL='b',
    TOK_COMMENT='#',
    TOK_OP='+'
};


enum{
#define OP2(X,Y)   ((X)|((Y)<<8))
#define OP3(X,Y,Z) ((X)|((Y)<<8)|((Z)<<16))

    OP_L_PAR=    '(', OP_R_PAR=    ')', OP_L_BRK=    '[', OP_R_BRK=    ']',
    OP_L_BRC=    '{', OP_R_BRC=    '}', OP_COMMA=    ',', OP_SCOLON=    ';',

    OP_PLUS=    '+', OP_MINUS=    '-', OP_MUL=    '*', OP_DIV=    '/',
    OP_MOD=        '%', OP_POW=    '^', OP_OR=     '|', OP_AND=    '&',
    /*OP_NOT=    '~',*/ OP_NOT=    '!', OP_ASGN=    '=', OP_LT=        '<',
    OP_GT=        '>', OP_DOT=    '.', OP_COLON=    ':', OP_QMARK=    '?',
    OP_AT=        '@',
    OP_NEXTLINE='\n',OP_EOF=    -1,
    
    OP_INC=        OP2('+','+'),         OP_DEC=    OP2('-','-'),
    OP_LSHIFT=    OP2('<','<'),          OP_RSHIFT=    OP2('>','>'),
    OP_AS_INC=    OP2('+','='),          OP_AS_DEC= OP2('-','='),
    OP_AS_MUL=    OP2('*','='),          OP_AS_DIV= OP2('/','='),
    OP_AS_MOD=    OP2('%','='),          OP_AS_POW= OP2('^','='),

/*    AS_OR=        OP2('|','='),         AS_AND=    OP2('&','='), */
    OP_EQ=        OP2('=','='),          OP_NE=        OP2('!','='),
    OP_LE=        OP2('<','='),          OP_GE=        OP2('>','=')
    
/*    L_AND=        OP2('&','&'), L_OR=        OP2('|','|'),
    L_XOR=        OP2('^','^'), */

/*    AsLShift=    OP3('<','<','='),
    AsRShift=    OP3('>','>','='), */
        
#undef OP2
#undef OP3
};


typedef struct{
    int type;
    int line;
    union{
        bool bval;
        long lval;
        double dval;
        char cval;
        char *sval;
        int opval;
    } u;
} Token;

#define TOK_INIT {0, 0, {0}}


extern void tok_free(Token*tok);
extern void tok_init(Token*tok);


/* */


enum{
    TOKZ_IGNORE_NEXTLINE=0x1,
    TOKZ_READ_COMMENTS=0x2,
    TOKZ_PARSER_INDENT_MODE=0x04,
    TOKZ_ERROR_TOLERANT=0x8,
    TOKZ_READ_FROM_BUFFER=0x10,
    TOKZ_DEFAULT_OPTION=0x20
};


enum{
    E_TOKZ_UNEXPECTED_EOF=1,
    E_TOKZ_UNEXPECTED_EOL,
    E_TOKZ_EOL_EXPECTED,
    E_TOKZ_INVALID_CHAR,
    E_TOKZ_TOOBIG,
    E_TOKZ_NUMFMT,
    E_TOKZ_NUM_JUNK,
    E_TOKZ_NOTINT,
    E_TOKZ_RANGE,
    E_TOKZ_MULTICHAR,
    
    E_TOKZ_TOKEN_LIMIT,
    E_TOKZ_UNKNOWN_OPTION,
    E_TOKZ_SYNTAX,
    E_TOKZ_INVALID_ARGUMENT,
    E_TOKZ_EOS_EXPECTED,
    E_TOKZ_TOO_FEW_ARGS,
    E_TOKZ_TOO_MANY_ARGS,
    E_TOKZ_MAX_NEST,
    E_TOKZ_IDENTIFIER_EXPECTED,
    
    E_TOKZ_LBRACE_EXPECTED
};


struct _ConfOpt;

typedef struct _Tokenizer_FInfo{
    FILE *file;
    char *name;
    int line;
    int ungetc;
    Token ungettok;
} Tokenizer_FInfo;

typedef struct _Tokenizer_Buffer{
        char *data;
        int len;
        int pos;
} Tokenizer_Buffer;
    
typedef struct _Tokenizer{
    FILE *file;
    char *name;
    int line;
    int ungetc;
    Token ungettok;
    
    Tokenizer_Buffer buffer;

    int flags;
    const struct _ConfOpt **optstack;
    int nest_lvl;
    void *user_data;

    int filestack_n;
    Tokenizer_FInfo *filestack;
    
    char **includepaths;
} Tokenizer;


extern Tokenizer *tokz_open(const char *fname);
extern Tokenizer *tokz_open_file(FILE *file, const char *fname);
extern Tokenizer *tokz_prepare_buffer(char *buffer, int len);
extern void tokz_close(Tokenizer *tokz);
extern bool tokz_get_token(Tokenizer *tokz, Token *tok);
extern void tokz_unget_token(Tokenizer *tokz, Token *tok);
extern void tokz_warn_error(const Tokenizer *tokz, int line, int e);
extern void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...);

extern bool tokz_pushf(Tokenizer *tokz, const char *fname);
extern bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname);
extern bool tokz_popf(Tokenizer *tokz);

extern void tokz_set_includepaths(Tokenizer *tokz, char **paths);

#endif /* LIBTU_TOKENIZER_H */