SHOGUN  v3.2.0
NGramTokenizer.h
浏览该文件的文档.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evangelos Anagnostopoulos
8  * Copyright (C) 2013 Evangelos Anagnostopoulos
9  */
10 
11 #ifndef _NGRAMTOKENIZER__H__
12 #define _NGRAMTOKENIZER__H__
13 
14 #include <shogun/lib/Tokenizer.h>
15 
16 namespace shogun
17 {
18 class CTokenizer;
19 
24 {
25 public:
30  CNGramTokenizer(int32_t ns=3);
31 
36  CNGramTokenizer(const CNGramTokenizer& orig);
37 
39  virtual ~CNGramTokenizer() {}
40 
45  virtual void set_text(SGVector<char> txt);
46 
52  virtual bool has_next();
53 
60  virtual index_t next_token_idx(index_t& start);
61 
67  virtual const char* get_name() const;
68 
69  virtual CNGramTokenizer* get_copy();
70 
71 private:
72  void init();
73 
74 protected:
75 
77  int32_t n;
78 
81 };
82 }
83 #endif /* _NGRAMTOKENIZER__H__ */
84 
The class CNGramTokenizer is used to tokenize a SGVector<char> into n-grams.
virtual const char * get_name() const
int32_t index_t
Definition: common.h:60
The class CTokenizer acts as a base class in order to implement tokenizers. Sub-classes must implemen...
Definition: Tokenizer.h:27
virtual void set_text(SGVector< char > txt)
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:16
virtual index_t next_token_idx(index_t &start)
CNGramTokenizer(int32_t ns=3)
virtual CNGramTokenizer * get_copy()

SHOGUN Machine Learning Toolbox - Documentation