Mercurial > repos > libtu-3 / file comparison

comparison: tokenizer.c

tokenizer.c

changeset 2
e14a1aba4c56
parent 1
6e704fc09528
child 3
b1fbfab67908
equal deleted inserted replaced
1:6e704fc09528 2:e14a1aba4c56
39 DUMMY_TR("Invalid argument"), /* E_TOKZ_INVALID_ARGUMENT */ 39 DUMMY_TR("Invalid argument"), /* E_TOKZ_INVALID_ARGUMENT */
40 DUMMY_TR("End of statement expected"), /* E_TOKZ_EOS_EXPECTED */ 40 DUMMY_TR("End of statement expected"), /* E_TOKZ_EOS_EXPECTED */
41 DUMMY_TR("Too few arguments"), /* E_TOKZ_TOO_FEW_ARGS */ 41 DUMMY_TR("Too few arguments"), /* E_TOKZ_TOO_FEW_ARGS */
42 DUMMY_TR("Too many arguments"), /* E_TOKZ_TOO_MANY_ARGS */ 42 DUMMY_TR("Too many arguments"), /* E_TOKZ_TOO_MANY_ARGS */
43 DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */ 43 DUMMY_TR("Maximum section nestin level exceeded"), /* E_TOK_Z_MAX_NEST */
44 DUMMY_TR("Unexpected end of statement"), /* E_TOKZ_UNEXPECTED_EOS */
45 DUMMY_TR("Identifier expected"), /* E_TOKZ_IDENTIFIER_EXPECTED */ 44 DUMMY_TR("Identifier expected"), /* E_TOKZ_IDENTIFIER_EXPECTED */
45 DUMMY_TR("Starting brace ('{') expected"), /* E_TOKZ_LBRACE_EXPECTED */
46 }; 46 };
47 47
48 48
49 /* */ 49 /* */
50 50
51 #define STRBLEN 32 51 #define STRBLEN 32
52 52
53 #define STRING_DECL(X) char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0 53 #define STRING_DECL(X) int err=0; char* X=NULL; char X##_tmp[STRBLEN]; int X##_tmpl=0
54 #define STRING_DECL_P(X, P) char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1 54 #define STRING_DECL_P(X, P) int err=0; char* X=NULL; char X##_tmp[STRBLEN]=P; int X##_tmpl=sizeof(P)-1
55 #define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) return -ENOMEM;} 55 #define STRING_APPEND(X, C) {if(!_string_append(&X, X##_tmp, &X##_tmpl, c)) err=-ENOMEM;}
56 #define STRING_FREE(X) if(X!=NULL) free(X) 56 #define STRING_FREE(X) if(X!=NULL) free(X)
57 #define STRING_FINISH(X) {if(!_string_finish(&X, X##_tmp, X##_tmpl)) return -ENOMEM;} 57 #define STRING_FINISH(X) {if(err!=0) return err; if(!_string_finish(&X, X##_tmp, X##_tmpl)) err=-ENOMEM;}
58 58
59 59
60 static bool _string_append(char **p, char *tmp, int *tmplen, char c) 60 static bool _string_append(char **p, char *tmp, int *tmplen, char c)
61 { 61 {
62 char *tmp2; 62 char *tmp2;
573 573
574 assert(tokz->file!=NULL); 574 assert(tokz->file!=NULL);
575 575
576 tok_free(tok); 576 tok_free(tok);
577 577
578 if(!TOK_IS_INVALID(&(tokz->ungettok))){
579 *tok=tokz->ungettok;
580 tokz->ungettok.type=TOK_INVALID;
581 return TRUE;
582 }
583
578 while(1){ 584 while(1){
579 585
580 e=0; 586 e=0;
581 587
582 do{ 588 do{
605 c=GETCH(); 611 c=GETCH();
606 if(c==EOF){ 612 if(c==EOF){
607 TOK_SET_OP(tok, OP_EOF); 613 TOK_SET_OP(tok, OP_EOF);
608 return FALSE; 614 return FALSE;
609 } 615 }
610 if(!isspace(c)){ 616 if(!isspace(c) && e==0){
611 tokz_warn_error(tokz, tokz->line, E_TOKZ_EOL_EXPECTED); 617 e=E_TOKZ_EOL_EXPECTED;
612 return FALSE; 618 tokz_warn_error(tokz, tokz->line, e);
619 if(!(tokz->flags&TOKZ_ERROR_TOLERANT))
620 return FALSE;
613 } 621 }
614 }while(c!='\n'); 622 }while(c!='\n');
615 623
616 INC_LINE(); 624 INC_LINE();
617 continue; 625 continue;
625 } 633 }
626 634
627 continue; 635 continue;
628 636
629 case '/': 637 case '/':
630 { 638 c2=GETCH();
631 c2=GETCH(); 639
632 640 if(c2=='='){
633 if(c2=='='){ 641 TOK_SET_OP(tok, OP_AS_DIV);
634 TOK_SET_OP(tok, OP_AS_DIV); 642 return TRUE;
635 return TRUE;
636 }
637
638 if(c2!='*'){
639 UNGETCH(c2);
640 TOK_SET_OP(tok, OP_DIV);
641 return TRUE;
642 }
643
644 if(tokz->flags&TOKZ_READ_COMMENTS){
645 e=scan_c_comment(tok, tokz);
646 break;
647 }else if((e=skip_c_comment(tokz))){
648 break;
649 }
650
651 continue;
652 } 643 }
644
645 if(c2!='*'){
646 UNGETCH(c2);
647 TOK_SET_OP(tok, OP_DIV);
648 return TRUE;
649 }
650
651 if(tokz->flags&TOKZ_READ_COMMENTS){
652 e=scan_c_comment(tok, tokz);
653 break;
654 }else if((e=skip_c_comment(tokz))){
655 break;
656 }
657
658 continue;
653 659
654 case '\"': 660 case '\"':
655 e=scan_string(tok, tokz, TRUE); 661 e=scan_string(tok, tokz, TRUE);
656 break; 662 break;
657 663
678 return FALSE; 684 return FALSE;
679 } 685 }
680 } 686 }
681 687
682 688
689 void tokz_unget_token(Tokenizer *tokz, Token *tok)
690 {
691 tok_free(&(tokz->ungettok));
692 tokz->ungettok=*tok;
693 tok->type=TOK_INVALID;
694 }
695
696
683 /* 697 /*
684 * File open 698 * File open
685 */ 699 */
686 700
687 static bool do_tokz_pushf(Tokenizer *tokz) 701 static bool do_tokz_pushf(Tokenizer *tokz)
699 713
700 finfo->file=tokz->file; 714 finfo->file=tokz->file;
701 finfo->name=tokz->name; 715 finfo->name=tokz->name;
702 finfo->line=tokz->line; 716 finfo->line=tokz->line;
703 finfo->ungetc=tokz->ungetc; 717 finfo->ungetc=tokz->ungetc;
704 718 finfo->ungettok=tokz->ungettok;
719
705 return TRUE; 720 return TRUE;
706 } 721 }
707 722
708 723
709 bool tokz_pushf_file(Tokenizer *tokz, FILE *file) 724 bool tokz_pushf_file(Tokenizer *tokz, FILE *file)
719 } 734 }
720 735
721 tokz->file=file; 736 tokz->file=file;
722 tokz->name=NULL; 737 tokz->name=NULL;
723 tokz->line=1; 738 tokz->line=1;
724 tokz->ungetc=-1; 739 tokz->ungetc=-1;
740 tokz->ungettok.type=TOK_INVALID;
725 741
726 return TRUE; 742 return TRUE;
727 } 743 }
728 744
729 745
774 } 790 }
775 791
776 tokz->file=NULL; 792 tokz->file=NULL;
777 tokz->name=NULL; 793 tokz->name=NULL;
778 tokz->line=1; 794 tokz->line=1;
779 tokz->ungetc=-1; 795 tokz->ungetc=-1;
796 tokz->ungettok.type=TOK_INVALID;
780 tokz->flags=0; 797 tokz->flags=0;
781 tokz->optstack=NULL; 798 tokz->optstack=NULL;
782 tokz->nest_lvl=0; 799 tokz->nest_lvl=0;
783 tokz->filestack_n=0; 800 tokz->filestack_n=0;
784 tokz->filestack=NULL; 801 tokz->filestack=NULL;
837 854
838 tokz->file=finfo->file; 855 tokz->file=finfo->file;
839 tokz->name=finfo->name; 856 tokz->name=finfo->name;
840 tokz->line=finfo->line; 857 tokz->line=finfo->line;
841 tokz->ungetc=finfo->ungetc; 858 tokz->ungetc=finfo->ungetc;
842 859 tokz->ungettok=finfo->ungettok;
860
843 if(tokz->filestack_n==0){ 861 if(tokz->filestack_n==0){
844 free(tokz->filestack); 862 free(tokz->filestack);
845 tokz->filestack=NULL; 863 tokz->filestack=NULL;
846 }else if(shrink){ 864 }else if(shrink){
847 finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo, 865 finfo=REALLOC_N(tokz->filestack, Tokenizer_FInfo,
869 887
870 if(tokz->file!=NULL) 888 if(tokz->file!=NULL)
871 fclose(tokz->file); 889 fclose(tokz->file);
872 if(tokz->name!=NULL) 890 if(tokz->name!=NULL)
873 free(tokz->name); 891 free(tokz->name);
874 892 tok_free(&(tokz->ungettok));
893
875 free(tokz); 894 free(tokz);
876 } 895 }
877 896
878 897
879 898
880 /* */ 899 /* */
881 900
882 901
883 void tok_free(Token *tok) 902 void tok_free(Token *tok)
884 { 903 {
885 if(TOK_IS_STRING(tok)) 904 if(TOK_IS_STRING(tok) || TOK_IS_IDENT(tok) || TOK_IS_COMMENT(tok)){
886 free(TOK_STRING_VAL(tok)); 905 if(TOK_STRING_VAL(tok)!=NULL)
906 free(TOK_STRING_VAL(tok));
907 }
887 908
888 tok->type=TOK_INVALID; 909 tok->type=TOK_INVALID;
889 } 910 }
890 911
891 912

Mercurial Repository: libtu-3

mercurial