tokenizer.c

/*
 * libtu/tokenizer.c
 *
 * Copyright (c) Tuomo Valkonen 1999-2002. 
 *
 * You may distribute and modify this library under the terms of either
 * the Clarified Artistic License or the GNU LGPL, version 2.1 or later.
 */

#include <errno.h>
#include <stdio.h>
#include <ctype.h>
#include <limits.h>
#include <assert.h>
#include <math.h>
#include <string.h>

#include <libtu/tokenizer.h>
#include <libtu/misc.h>
#include <libtu/output.h>


static const char *errors[]={
    DUMMY_TR("(no error)"),
    DUMMY_TR("Unexpected end of file"),                /* E_TOKZ_UNEXPECTED_EOF */
    DUMMY_TR("Unexpected end of line"),                /* E_TOKZ_UNEXPECTED_EOL */
    DUMMY_TR("End of line expected"),                /* E_TOKZ_EOL_EXPECTED */
    DUMMY_TR("Invalid character"),                    /* E_TOKZ_INVALID_CHAR*/
    DUMMY_TR("Numeric constant too big"),            /* E_TOKZ_TOOBIG */
    DUMMY_TR("Invalid numberic format"),            /* E_TOKZ_NUMFMT */
    DUMMY_TR("Junk after numeric constant"),        /* E_TOKZ_NUM_JUNK */
    DUMMY_TR("Not an integer"),                        /* E_TOKZ_NOTINT */
    DUMMY_TR("Numeric constant out of range"),        /* E_TOKZ_RANGE */
    DUMMY_TR("Multi-character character constant"),    /* E_TOKZ_MULTICHAR */
    DUMMY_TR("Token/statement limit reached"),        /* E_TOKZ_TOKEN_LIMIT */
    DUMMY_TR("Unknown option"),                        /* E_TOKZ_UNKONWN_OPTION */
    DUMMY_TR("Syntax error"),                        /* E_TOKZ_SYNTAX */
    DUMMY_TR("Invalid argument"),                    /* E_TOKZ_INVALID_ARGUMENT */
    DUMMY_TR("End of statement expected"),            /* E_TOKZ_EOS_EXPECTED */
    DUMMY_TR("Too few arguments"),                    /* E_TOKZ_TOO_FEW_ARGS */
    DUMMY_TR("Too many arguments"),                    /* E_TOKZ_TOO_MANY_ARGS */
    DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */
    DUMMY_TR("Identifier expected"),                /* E_TOKZ_IDENTIFIER_EXPECTED */
    DUMMY_TR("Starting brace ('{') expected"),        /* E_TOKZ_LBRACE_EXPECTED */
};


/* */

#define STRBLEN 32

#define STRING_DECL(X) int err=0; char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0
#define STRING_DECL_P(X, P) int err=0; char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1
#define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) err=-ENOMEM;}
#define STRING_FREE(X) if(X!=NULL) free(X)
#define STRING_FINISH(X) {if(err!=0) return err; if(!_string_finish(&X, X##_tmp, X##_tmpl)) err=-ENOMEM;}


static bool _string_append(char **p, char *tmp, int *tmplen, char c)
{
    char *tmp2;
    
    if(*tmplen==STRBLEN-1){
        tmp[STRBLEN-1]='\0';
        if(*p!=NULL){
            tmp2=scat(*p, tmp);
            free(*p);
            *p=tmp2;
        }else{
            *p=scopy(tmp);
        }
        *tmplen=1;
        tmp[0]=c;
        return *p!=NULL;
    }else{
        tmp[(*tmplen)++]=c;
        return TRUE;
    }
}


static bool _string_finish(char **p, char *tmp, int tmplen)
{
    char *tmp2;
    
    if(tmplen==0){
        if(*p==NULL)
            *p=scopy("");
    }else{
        tmp[tmplen]='\0';
        if(*p!=NULL){
            tmp2=scat(*p, tmp);
            free(*p);
            *p=tmp2;
        }else{
            *p=scopy(tmp);
        }
    }
    return *p!=NULL;
}


/* */


#define INC_LINE() tokz->line++
#define GETCH() _getch(tokz)
#define UNGETCH(C) _ungetch(tokz, C)

static int _getch(Tokenizer *tokz)
{
    int c;
    
    if(tokz->ungetc!=-1){
        c=tokz->ungetc;
        tokz->ungetc=-1;
    }else if (tokz->flags&TOKZ_READ_FROM_BUFFER) {
        assert(tokz->buffer.data!=NULL);
        if (tokz->buffer.pos==tokz->buffer.len)
            c=EOF;
        else
            c=tokz->buffer.data[tokz->buffer.pos++];
    }else{
        c=getc(tokz->file);
    }

    return c;
}


static void _ungetch(Tokenizer *tokz, int c)
{
    tokz->ungetc=c;
}


/* */


static int scan_line_comment(Token *tok, Tokenizer *tokz)
{
    STRING_DECL_P(s, "#");
    int c;

    c=GETCH();
                
    while(c!='\n' && c!=EOF){
        STRING_APPEND(s, c);
        c=GETCH();
    }

    UNGETCH(c);

    STRING_FINISH(s);
    
    TOK_SET_COMMENT(tok, s);
    
    return 0;
}


static int skip_line_comment(Tokenizer *tokz)
{
    int c;
    
    do{
        c=GETCH();
    }while(c!='\n' && c!=EOF);

    UNGETCH(c);
        
    return 0;
}


/* */


static int scan_c_comment(Token *tok, Tokenizer *tokz)
{
    STRING_DECL_P(s, "/*");
    int c;
    int st=0;
    
    while(1){
        c=GETCH();
        
        if(c==EOF){
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOF;
        }
        
        STRING_APPEND(s, c);
        
        if(c=='\n'){
            INC_LINE();
        }else if(st==0 && c=='*'){
            st=1;
        }else if(st==1){
            if(c=='/')
                break;
            st=0;
        }
    }

    STRING_FINISH(s);

    TOK_SET_COMMENT(tok, s);

    return 0;
}


static int skip_c_comment(Tokenizer *tokz)
{
    int c;
    int st=0;
    
    while(1){
        c=GETCH();
        
        if(c==EOF)
            return E_TOKZ_UNEXPECTED_EOF;
        
        if(c=='\n')
            INC_LINE();
        else if(st==0 && c=='*')
            st=1;
        else if(st==1){
            if(c=='/')
                break;
            st=0;
        }
    }
    
    return 0;
}


/* */


static int scan_char_escape(Tokenizer *tokz)
{
    static char* special_chars="nrtbae";
    static char* specials="\n\r\t\b\a\033";
    int base, max;
    int i ,c;
    
    c=GETCH();
    
    for(i=0;special_chars[i];i++){
        if(special_chars[i]==c)
            return specials[c];
    }
    
    if(c=='x' || c=='X'){
        base=16;max=2;i=0;
    }else if(c=='d' || c=='D'){
        base=10;max=3;i=0;
    }else if(c=='8' || c=='9'){
        base=10;max=2;i=c-'0';
    }else if('0'<=c && c<='7'){
        base=8;max=2;i=c-'0';
    }else if(c=='\n'){
        UNGETCH(c);
        return -2;
    }else{
        return c;
    }
    
        
    while(--max>=0){
        c=GETCH();
        
        if(c==EOF)
            return EOF;
        
        if(c=='\n'){
            UNGETCH(c);
            return -2;
        }
        
        if(base==16){
            if(!isxdigit(c))
                break;
            
            i<<=4;
            
            if(isdigit(c))
                i+=c-'0';
            else if(i>='a')
                i+=0xa+c-'a';
            else
                i+=0xa+c-'a';
            
        }else if(base==10){
            if(!isdigit(c))
                break;
            i*=10;
            i+=c-'0';
        }else{
            if(c<'0' || c>'7')
                break;
            i<<=3;
            i+=c-'0';
        }
    }
    
    if(max>=0)
        UNGETCH(c);

    return i;
}


/* */


static int scan_string(Token *tok, Tokenizer *tokz, bool escapes)
{
    STRING_DECL(s);
    int c;

    while(1){    
        c=GETCH();
        
        if(c=='"')
            break;
        
        if(c=='\n'){
            UNGETCH(c);
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOL;
        }
        
        if(c=='\\' && escapes){
            c=scan_char_escape(tokz);
            if(c==-2){
                STRING_FREE(s);
                return E_TOKZ_UNEXPECTED_EOL;
            }
        }
        
        if(c==EOF){
            STRING_FREE(s);
            return E_TOKZ_UNEXPECTED_EOF;
        }
        
        STRING_APPEND(s, c);
    }
    
    STRING_FINISH(s);
    
    TOK_SET_STRING(tok, s);

    return 0;
}


/* */


static int scan_char(Token *tok, Tokenizer *tokz)
{
    int c, c2;
    
    c=GETCH();
    
    if(c==EOF)
        return E_TOKZ_UNEXPECTED_EOF;
    
    if(c=='\n')
        return E_TOKZ_UNEXPECTED_EOL;

    if(c=='\\'){
        c=scan_char_escape(tokz);    
        
        if(c==EOF)
            return E_TOKZ_UNEXPECTED_EOF;
        
        if(c==-2)
            return E_TOKZ_UNEXPECTED_EOL;
    }
    
    c2=GETCH();
    
    if(c2!='\'')
        return E_TOKZ_MULTICHAR;
    
    TOK_SET_CHAR(tok, c);
    
    return 0;
}


/* */


#define START_IDENT(X) (isalpha(X) || X=='_' || X=='$')


static int scan_identifier(Token *tok, Tokenizer *tokz, int c)
{
    STRING_DECL(s);
    
    do{
        STRING_APPEND(s, c);
        c=GETCH();
    }while(isalnum(c) || c=='_' || c=='$');
    
    UNGETCH(c);
    
    STRING_FINISH(s);
    
    TOK_SET_IDENT(tok, s);

    return 0;
}

#define NP_SIMPLE_IMPL
#include "np/numparser2.h"
#include "np/np-conv.h"


static int scan_number(Token *tok, Tokenizer *tokz, int c)
{
    NPNum num=NUM_INIT;
    int e;
    
    if((e=parse_number(&num, tokz, c)))
        return e;
    
    if(num.type==NPNUM_INT){
        long l;
        if((e=num_to_long(&l, &num, TRUE)))
            return e;
    
        TOK_SET_LONG(tok, l);
    }else if(num.type==NPNUM_FLOAT){
          double d;
          if((e=num_to_double(&d, &num)))
              return e;
            
        TOK_SET_DOUBLE(tok, d);
    }else{
        return E_TOKZ_NUMFMT;
    }

    return 0;
}


/* */


static uchar op_map[]={
    0x00,        /* ________ 0-7     */
    0x00,        /* ________ 8-15    */
    0x00,        /* ________ 16-23   */
    0x00,        /* ________ 24-31   */
    0x62,        /* _!___%&_ 32-39   */
    0xff,        /* ()*+,-./ 40-47   */
    0x00,        /* ________ 48-55   */
    0xfc,        /* __:;<=>? 56-63   */
    0x01,        /* @_______ 64-71   */
    0x00,        /* ________ 72-79   */
    0x00,        /* ________ 80-87   */
    0x78,        /* ___[_]^_ 88-95   */
    0x00,        /* ________ 96-103  */
    0x00,        /* ________ 104-111 */
    0x00,        /* ________ 112-119 */
    0x38        /* ___{|}__ 120-127 */
};


static bool map_isset(uchar *map, uint ch)
{
    if(ch>127)
        return FALSE;

    return map[ch>>3]&(1<<(ch&7));
}


static bool is_opch(uint ch)
{
    return map_isset(op_map, ch);
}


static int scan_op(Token *tok, Tokenizer *tokz,  int c)
{
    int c2;
    int op=-1;
    
    /* Quickly check it is an operator character */
    if(!is_opch(c))
        return E_TOKZ_INVALID_CHAR;

    switch(c){
    case '+':
    case '-':
    case '*':
/*    case '/':     Checked elsewhere */
    case '%':
    case '^':
    case '!':
    case '=':
    case '<':
    case '>':
        c2=GETCH();
        if(c2=='='){
            op=c|(c2<<8);
        }else if(c2==c && (c2!='%' && c2!='!' && c2!='*')){
            if(c=='<' || c=='>'){
                int c3=GETCH();
                if(c3=='='){
                    op=c|(c2<<8)|(c3<<16);
                }else{
                    UNGETCH(c3);
                    op=c|(c2<<8);
                }
            }else{
                op=c|(c2<<8);
            }
        }else{
            UNGETCH(c2);
            op=c;
        }
        break;
        
    /* It is already known that it is a operator so these are not needed
    case ':':
    case '~':
    case '?':
    case '.':
    case ';';
    case '{':
    case '}':
    case '@':        
    case '|':
    case '&':
    */
    default:
        op=c;
    }
    
    TOK_SET_OP(tok, op);

    return 0;
}


/* */


void tokz_warn(const Tokenizer *tokz, int line, const char *fmt, ...)
{
    va_list args;
    
    va_start(args, fmt);
    
    if(tokz!=NULL)
        warn_obj_line_v(tokz->name, line, fmt, args);
    else
        warn(fmt, args);
    
    va_end(args);
}


void tokz_warn_error(const Tokenizer *tokz, int line, int e)
{
    if(e==E_TOKZ_UNEXPECTED_EOF)
        line=0;
    
    if(e<0)
        tokz_warn(tokz, line, "%s", strerror(-e));
    else
        tokz_warn(tokz, line, "%s", TR(errors[e]));
}


bool tokz_get_token(Tokenizer *tokz, Token *tok)
{
    int c, c2, e;
    
    if (!(tokz->flags&TOKZ_READ_FROM_BUFFER))
    assert(tokz->file!=NULL);
    
    tok_free(tok);
    
    if(!TOK_IS_INVALID(&(tokz->ungettok))){
        *tok=tokz->ungettok;
        tokz->ungettok.type=TOK_INVALID;
        return TRUE;
    }

    while(1){
    
        e=0;
        
        do{
            c=GETCH();
        }while(c!='\n' && c!=EOF && isspace(c));
    
        tok->line=tokz->line;
    
        switch(c){
        case EOF:
            TOK_SET_OP(tok, OP_EOF);
            return TRUE;
            
        case '\n':
            INC_LINE();
            
            if(tokz->flags&TOKZ_IGNORE_NEXTLINE)
                continue;
            
            TOK_SET_OP(tok, OP_NEXTLINE);
            
            return TRUE;
            
        case '\\':
            do{
                c=GETCH();
                if(c==EOF){
                    TOK_SET_OP(tok, OP_EOF);
                    return FALSE;
                }
                if(!isspace(c) && e==0){
                    e=E_TOKZ_EOL_EXPECTED;
                    tokz_warn_error(tokz, tokz->line, e);
                    if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
                        return FALSE;
                }
            }while(c!='\n');
            
            INC_LINE();
            continue;

        case '#':
            if(tokz->flags&TOKZ_READ_COMMENTS){
                e=scan_line_comment(tok, tokz);
                break;
            }else if((e=skip_line_comment(tokz))){
                break;
            }
            
            continue;
            
        case '/':
            c2=GETCH();
            
            if(c2=='='){
                TOK_SET_OP(tok, OP_AS_DIV);
                return TRUE;
            }
            
            if(c2!='*'){
                UNGETCH(c2);
                TOK_SET_OP(tok, OP_DIV);
                return TRUE;
            }
            
            if(tokz->flags&TOKZ_READ_COMMENTS){
                e=scan_c_comment(tok, tokz);
                break;
            }else if((e=skip_c_comment(tokz))){
                break;
            }
            
            continue;
            
        case '\"':
            e=scan_string(tok, tokz, TRUE);
            break;

        case '\'':
            e=scan_char(tok, tokz);
            break;

        default: 
            if(('0'<=c && c<='9') || c=='-' || c=='+'){
                e=scan_number(tok, tokz, c);
                break;
            }

             if(START_IDENT(c))
                e=scan_identifier(tok, tokz, c);
            else
                e=scan_op(tok, tokz, c);
        }
        
        if(!e)
            return TRUE;
        
        tokz_warn_error(tokz, tokz->line, e);
        return FALSE;
    }
}


void tokz_unget_token(Tokenizer *tokz, Token *tok)
{
    tok_free(&(tokz->ungettok));    
    tokz->ungettok=*tok;
    tok->type=TOK_INVALID;
}


/*
 * File open
 */

static bool do_tokz_pushf(Tokenizer *tokz)
{
    Tokenizer_FInfo *finfo;
    
    finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
                    tokz->filestack_n, tokz->filestack_n+1);
    
    if(finfo==NULL)
        return FALSE;

    tokz->filestack=finfo;
    finfo=&(finfo[tokz->filestack_n++]);
    
    finfo->file=tokz->file;
    finfo->name=tokz->name;
    finfo->line=tokz->line;
    finfo->ungetc=tokz->ungetc;
    finfo->ungettok=tokz->ungettok;
    
    return TRUE;
}


bool tokz_pushf_file(Tokenizer *tokz, FILE *file, const char *fname)
{
    char *fname_copy=NULL;
    
    if(file==NULL)
        return FALSE;

    if(fname!=NULL){
        fname_copy=scopy(fname);
        if(fname_copy==NULL){
            warn_err();
            return FALSE;
        }
    }
    
    if(tokz->file!=NULL){
        if(!do_tokz_pushf(tokz)){
            warn_err();
            if(fname_copy!=NULL)
                free(fname_copy);
            return FALSE;
        }
    }
    
    tokz->file=file;
    tokz->name=fname_copy;
    tokz->line=1;
    tokz->ungetc=-1;    
    tokz->ungettok.type=TOK_INVALID;
    
    return TRUE;
}


bool tokz_pushf(Tokenizer *tokz, const char *fname)
{
    FILE *file;
    
    file=fopen(fname, "r");
    
    if(file==NULL){
        warn_err_obj(fname);
        return FALSE;
    }
    
    if(!tokz_pushf_file(tokz, file, fname)){
        fclose(file);
        return FALSE;
    }

    return TRUE;
}


static Tokenizer *tokz_create()
{
    Tokenizer*tokz;
    
    tokz=ALLOC(Tokenizer);
    
    if(tokz==NULL){
        warn_err();
        return NULL;
    }
    
    tokz->file=NULL;
    tokz->name=NULL;
    tokz->line=1;
    tokz->ungetc=-1;    
    tokz->ungettok.type=TOK_INVALID;
    tokz->flags=0;
    tokz->optstack=NULL;
    tokz->nest_lvl=0;
    tokz->filestack_n=0;
    tokz->filestack=NULL;
    tokz->buffer.data=0;
    tokz->buffer.len=0;
    tokz->buffer.pos=0;
    
    return tokz;
}

    
Tokenizer *tokz_open(const char *fname)
{ 
    Tokenizer *tokz;
    
    tokz=tokz_create();
    
    if(!tokz_pushf(tokz, fname)){
        free(tokz);
        return NULL;
    }
    
    return tokz;
}


Tokenizer *tokz_open_file(FILE *file, const char *fname)
{
    Tokenizer *tokz;
    
    tokz=tokz_create();
    
    if(!tokz_pushf_file(tokz, file, fname)){
        free(tokz);
        return NULL;
    }
    
    return tokz;
}

Tokenizer *tokz_prepare_buffer(char *buffer, int len)
{
    Tokenizer *tokz;
    char old=0;

    tokz=tokz_create();
    if(len>0){
        old=buffer[len-1];
        buffer[len-1]='\0';
    }

    tokz->flags|=TOKZ_READ_FROM_BUFFER;
    tokz->buffer.data=scopy(buffer);
    tokz->buffer.len=(len>0 ? (uint)len : strlen(tokz->buffer.data));
    tokz->buffer.pos=0;

    if(old>0)
        buffer[len-1]=old;

    return tokz;
}

/*
 * File close
 */

static bool do_tokz_popf(Tokenizer *tokz, bool shrink)
{
    Tokenizer_FInfo *finfo;
    
    if(tokz->filestack_n<=0)
        return FALSE;

    if(tokz->file!=NULL)
        fclose(tokz->file);
    if(tokz->name!=NULL)
        free(tokz->name);
    
    finfo=&(tokz->filestack[--tokz->filestack_n]);
    
    tokz->file=finfo->file;
    tokz->name=finfo->name;
    tokz->line=finfo->line;
    tokz->ungetc=finfo->ungetc;
    tokz->ungettok=finfo->ungettok;
    
    if(tokz->filestack_n==0){
        free(tokz->filestack);
        tokz->filestack=NULL;
    }else if(shrink){
        finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
                        tokz->filestack_n+1, tokz->filestack_n);
        if(finfo==NULL)
            warn_err();
        else
            tokz->filestack=finfo;
    }
    
    return TRUE;
}


bool tokz_popf(Tokenizer *tokz)
{
    return do_tokz_popf(tokz, TRUE);
}
    

void tokz_close(Tokenizer *tokz)
{
    while(tokz->filestack_n>0)
        do_tokz_popf(tokz, FALSE);
          
    if(tokz->file!=NULL)
        fclose(tokz->file);
    if(tokz->name!=NULL)
        free(tokz->name);
    tok_free(&(tokz->ungettok));
    
    free(tokz);
}


/* */


void tok_free(Token *tok)
{
    if(TOK_IS_STRING(tok) || TOK_IS_IDENT(tok) || TOK_IS_COMMENT(tok)){
        if(TOK_STRING_VAL(tok)!=NULL)
            free(TOK_STRING_VAL(tok));
    }
    
    tok->type=TOK_INVALID;
}


void tok_init(Token *tok)
{
    static Token dummy=TOK_INIT;
    
    memcpy(tok, &dummy, sizeof(*tok));
}