7#ifndef STANDARDTOKENIZERIMPL_H
8#define STANDARDTOKENIZERIMPL_H
#define LUCENE_CLASS(Name)
Definition: LuceneObject.h:24
Base class for all Lucene classes.
Definition: LuceneObject.h:31
Definition: StandardTokenizerImpl.h:14
int32_t yycolumn
The number of characters from the last newline up to the start of the matched text.
Definition: StandardTokenizerImpl.h:119
int32_t zzEndRead
EndRead marks the last character in the buffer, that has been read from input.
Definition: StandardTokenizerImpl.h:110
int32_t zzMarkedPos
The text position at the last accepting state.
Definition: StandardTokenizerImpl.h:98
int32_t zzCurrentPos
The current text position in the buffer.
Definition: StandardTokenizerImpl.h:104
void yyclose()
Closes the input stream.
static const int32_t ZZ_ACTION_LENGTH
Definition: StandardTokenizerImpl.h:41
int32_t zzLexicalState
The current lexical state.
Definition: StandardTokenizerImpl.h:92
static IntArray _ZZ_ROWMAP
Translates a state to a row index in the transition table.
Definition: StandardTokenizerImpl.h:49
static void ZZ_ATTRIBUTE_INIT()
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
static const wchar_t * ZZ_CMAP()
static const int32_t ZZ_ACTION_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:42
static const int32_t ZZ_NO_MATCH
Definition: StandardTokenizerImpl.h:70
static void ZZ_ROWMAP_INIT()
Translates a state to a row index in the transition table.
static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:79
static const int32_t ZZ_TRANS_LENGTH
Definition: StandardTokenizerImpl.h:61
static const int32_t * ZZ_ACTION()
static IntArray _ZZ_ACTION
Translates DFA states to action switch labels.
Definition: StandardTokenizerImpl.h:39
static void ZZ_CMAP_INIT()
Translates characters to character classes.
void reset(const ReaderPtr &r)
Resets the Tokenizer to a new Reader.
static const int32_t YYEOF
This character denotes the end of file.
Definition: StandardTokenizerImpl.h:129
static const int32_t ZZ_ROWMAP_LENGTH
Definition: StandardTokenizerImpl.h:51
ReaderPtr zzReader
The input device.
Definition: StandardTokenizerImpl.h:86
static const int32_t ZZ_BUFFERSIZE
Initial size of the lookahead buffer.
Definition: StandardTokenizerImpl.h:26
static IntArray _ZZ_ATTRIBUTE
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
Definition: StandardTokenizerImpl.h:76
static const int32_t ZZ_CMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:32
void zzScanError(int32_t errorCode)
Reports an error that occurred while scanning.
static const int32_t * ZZ_ATTRIBUTE()
String yytext()
Returns the text matched by the current regular expression.
static CharArray _ZZ_CMAP
Translates characters to character classes.
Definition: StandardTokenizerImpl.h:29
void yypushback(int32_t number)
Pushes the specified amount of characters back into the input stream.
void yyreset(const ReaderPtr &reader)
Resets the scanner to read from a new input stream. Does not close the old reader.
static const int32_t ZZ_ATTRIBUTE_LENGTH
Definition: StandardTokenizerImpl.h:78
static const int32_t ZZ_CMAP_LENGTH
Definition: StandardTokenizerImpl.h:31
int32_t zzState
The current state of the DFA.
Definition: StandardTokenizerImpl.h:89
bool zzAtBOL
zzAtBOL == true if the scanner is currently at the beginning of a line
Definition: StandardTokenizerImpl.h:122
bool zzAtEOF
zzAtEOF == true if the scanner is at the EOF
Definition: StandardTokenizerImpl.h:125
int32_t yylength()
Returns the length of the matched text region.
wchar_t yycharat(int32_t pos)
Returns the character at position pos from the matched text.
void getText(const TokenPtr &t)
Fills Lucene token with the current token text.
static const int32_t * ZZ_TRANS()
int32_t getNextToken()
Resumes scanning until the next regular expression is matched, the end of input is encountered or an ...
CharArray zzBuffer
This buffer contains the current text to be matched and is the source of the yytext() string.
Definition: StandardTokenizerImpl.h:95
int32_t yystate()
Returns the current lexical state.
static const int32_t ZZ_PUSHBACK_2BIG
Definition: StandardTokenizerImpl.h:71
int32_t zzPushbackPos
The text position at the last state to be included in yytext.
Definition: StandardTokenizerImpl.h:101
static void ZZ_ACTION_INIT()
Translates DFA states to action switch labels.
bool zzRefill()
Refills the input buffer.
virtual ~StandardTokenizerImpl()
int32_t _yychar
The number of characters up to the start of the matched text.
Definition: StandardTokenizerImpl.h:116
static IntArray _ZZ_TRANS
The transition table of the DFA.
Definition: StandardTokenizerImpl.h:59
static const int32_t ZZ_UNKNOWN_ERROR
Definition: StandardTokenizerImpl.h:69
int32_t zzStartRead
StartRead marks the beginning of the yytext() string in the buffer.
Definition: StandardTokenizerImpl.h:107
static const wchar_t ZZ_TRANS_PACKED_0[]
Definition: StandardTokenizerImpl.h:60
StandardTokenizerImpl(const ReaderPtr &in)
Creates a new scanner.
void getText(const TermAttributePtr &t)
Fills TermAttribute with the current token text.
static const wchar_t ZZ_ATTRIBUTE_PACKED_0[]
Definition: StandardTokenizerImpl.h:77
static const wchar_t * ZZ_ERROR_MSG[]
Definition: StandardTokenizerImpl.h:73
static const wchar_t ZZ_ROWMAP_PACKED_0[]
Definition: StandardTokenizerImpl.h:50
static const wchar_t ZZ_CMAP_PACKED[]
Definition: StandardTokenizerImpl.h:30
void yybegin(int32_t newState)
Enters a new lexical state.
static const int32_t YYINITIAL
Lexical states.
Definition: StandardTokenizerImpl.h:132
static const int32_t * ZZ_ROWMAP()
static void ZZ_TRANS_INIT()
The transition table of the DFA.
static const wchar_t ZZ_ACTION_PACKED_0[]
Definition: StandardTokenizerImpl.h:40
static const int32_t ZZ_TRANS_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:62
int32_t yyline
Number of newlines encountered up to the start of the matched text.
Definition: StandardTokenizerImpl.h:113
static const int32_t ZZ_ROWMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:52
Definition: AbstractAllTermDocs.h:12
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition: LuceneTypes.h:58
boost::shared_ptr< Token > TokenPtr
Definition: LuceneTypes.h:59
boost::shared_ptr< Reader > ReaderPtr
Definition: LuceneTypes.h:547