27 m_tokenizer=tokenizer;
29 if (m_tokenizer!=NULL)
40 if (m_tokenizer!=NULL)
60 for (
index_t i=start; i<end; i++)
62 result[i-start]=m_text[i];
76 for (
index_t i=start; i<end; i++)
78 result[i-start]=m_text[i];
80 result[end-start]=
'\0';
90 return (
bool) strtod(token.
vector, NULL);
95 #define READ_INT_METHOD(fname, convf, sg_type) \
96 sg_type CParser::fname() \
98 SGVector<char> token=read_cstring(); \
101 return (sg_type) convf(token.vector, NULL, 10); \
103 return (sg_type) 0L; \
108 #undef READ_INT_METHOD
110 #define READ_REAL_METHOD(fname, convf, sg_type) \
111 sg_type CParser::fname() \
113 SGVector<char> token=read_cstring(); \
116 return (sg_type) convf(token.vector, NULL); \
118 return (sg_type) 0L; \
135 #undef READ_REAL_METHOD
141 if (m_tokenizer!=NULL)
149 m_tokenizer=tokenizer;
151 if (m_tokenizer!=NULL)
void set_text(SGVector< char > text)
#define READ_REAL_METHOD(fname, convf, sg_type)
virtual void set_text(SGVector< char > txt)
virtual void skip_token()
virtual SGVector< char > read_string()
The class CTokenizer acts as a base class in order to implement tokenizers. Sub-classes must implemen...
virtual bool has_next()=0
void set_tokenizer(CTokenizer *tokenizer)
all of classes and functions are contained in the shogun namespace
virtual index_t next_token_idx(index_t &start)=0
virtual SGVector< char > read_cstring()
#define READ_INT_METHOD(fname, convf, sg_type)