comparison: tokenizer.c
tokenizer.c
- changeset 35
- 5a71d53d0228
- parent 21
- 6d4282804e73
- child 36
- 63cd573ffbcf
equal
deleted
inserted
replaced
| 34:828f3afd5c76 | 35:5a71d53d0228 |
|---|---|
| 110 int c; | 110 int c; |
| 111 | 111 |
| 112 if(tokz->ungetc!=-1){ | 112 if(tokz->ungetc!=-1){ |
| 113 c=tokz->ungetc; | 113 c=tokz->ungetc; |
| 114 tokz->ungetc=-1; | 114 tokz->ungetc=-1; |
| 115 }else if (tokz->flags&TOKZ_READ_FROM_BUFFER) { | |
| 116 assert(tokz->buffer.data!=NULL); | |
| 117 if (tokz->buffer.pos==tokz->buffer.len) | |
| 118 c=EOF; | |
| 119 else | |
| 120 c=tokz->buffer.data[tokz->buffer.pos++]; | |
| 115 }else{ | 121 }else{ |
| 116 c=getc(tokz->file); | 122 c=getc(tokz->file); |
| 117 } | 123 } |
| 118 | 124 |
| 119 return c; | 125 return c; |
| 576 | 582 |
| 577 bool tokz_get_token(Tokenizer *tokz, Token *tok) | 583 bool tokz_get_token(Tokenizer *tokz, Token *tok) |
| 578 { | 584 { |
| 579 int c, c2, e; | 585 int c, c2, e; |
| 580 | 586 |
| 587 if (!(tokz->flags&TOKZ_READ_FROM_BUFFER)) | |
| 581 assert(tokz->file!=NULL); | 588 assert(tokz->file!=NULL); |
| 582 | 589 |
| 583 tok_free(tok); | 590 tok_free(tok); |
| 584 | 591 |
| 585 if(!TOK_IS_INVALID(&(tokz->ungettok))){ | 592 if(!TOK_IS_INVALID(&(tokz->ungettok))){ |
| 802 tokz->flags=0; | 809 tokz->flags=0; |
| 803 tokz->optstack=NULL; | 810 tokz->optstack=NULL; |
| 804 tokz->nest_lvl=0; | 811 tokz->nest_lvl=0; |
| 805 tokz->filestack_n=0; | 812 tokz->filestack_n=0; |
| 806 tokz->filestack=NULL; | 813 tokz->filestack=NULL; |
| 814 tokz->buffer.data=0; | |
| 815 tokz->buffer.len=0; | |
| 816 tokz->buffer.pos=0; | |
| 807 | 817 |
| 808 return tokz; | 818 return tokz; |
| 809 } | 819 } |
| 810 | 820 |
| 811 | 821 |
| 836 } | 846 } |
| 837 | 847 |
| 838 return tokz; | 848 return tokz; |
| 839 } | 849 } |
| 840 | 850 |
| 851 Tokenizer *tokz_prepare_buffer(char *buffer, int len) | |
| 852 { | |
| 853 Tokenizer *tokz; | |
| 854 char old=0; | |
| 855 | |
| 856 tokz=tokz_create(); | |
| 857 if (len>0) { | |
| 858 old=buffer[len-1]; | |
| 859 buffer[len-1]='\0'; | |
| 860 } | |
| 861 | |
| 862 tokz->flags|=TOKZ_READ_FROM_BUFFER; | |
| 863 tokz->buffer.data=scopy(buffer); | |
| 864 tokz->buffer.len=len>0 ? len : strlen(tokz->buffer.data); | |
| 865 tokz->buffer.pos=0; | |
| 866 | |
| 867 if (old>0) | |
| 868 buffer[len-1]=old; | |
| 869 | |
| 870 return tokz; | |
| 871 } | |
| 841 | 872 |
| 842 /* | 873 /* |
| 843 * File close | 874 * File close |
| 844 */ | 875 */ |
| 845 | 876 |
