1 /* -*- c-basic-offset: 2 -*- */
2 /*
3   Copyright(C) 2014-2018  Brazil
4   Copyright(C) 2018-2021  Sutou Kouhei <kou@clear-code.com>
5 
6   This library is free software; you can redistribute it and/or
7   modify it under the terms of the GNU Lesser General Public
8   License version 2.1 as published by the Free Software Foundation.
9 
10   This library is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13   Lesser General Public License for more details.
14 
15   You should have received a copy of the GNU Lesser General Public
16   License along with this library; if not, write to the Free Software
17   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
18 */
19 module groonga_d.token;
20 
21 
22 private static import groonga_d.groonga;
23 
24 extern(C):
25 nothrow @nogc:
26 
27 /*
28  * grn_tokenize_mode describes propose for tokenization.
29  *
30  * `GRN_TOKENIZE_GET`: Tokenize for search.
31  *
32  * `GRN_TOKENIZE_ADD`: Tokenize for adding token to index.
33  *
34  * `GRN_TOKENIZE_DELETE`: Tokenize for deleting token from index.
35  *
36  * @since 4.0.8
37  */
38 enum grn_tokenize_mode
39 {
40 	GRN_TOKENIZE_GET = 0,
41 	GRN_TOKENIZE_ADD,
42 	GRN_TOKENIZE_DELETE,
43 	GRN_TOKENIZE_ONLY,
44 }
45 
46 //Declaration name in C language
47 enum
48 {
49 	GRN_TOKENIZE_GET = .grn_tokenize_mode.GRN_TOKENIZE_GET,
50 	GRN_TOKENIZE_ADD = .grn_tokenize_mode.GRN_TOKENIZE_ADD,
51 	GRN_TOKENIZE_DELETE = .grn_tokenize_mode.GRN_TOKENIZE_DELETE,
52 	GRN_TOKENIZE_ONLY = .grn_tokenize_mode.GRN_TOKENIZE_ONLY,
53 }
54 
55 /*
56   grn_token_mode describes propose for tokenization.
57 
58   `GRN_TOKEN_GET`: Tokenization for search.
59 
60   `GRN_TOKEN_ADD`: Tokenization for adding token to index.
61 
62   `GRN_TOKEN_DEL`: Tokenization for deleting token from index.
63 
64   @since 4.0.7
65   @deprecated since 4.0.8. Use grn_tokenize_mode instead.
66  */
67 alias grn_token_mode = grn_tokenize_mode;
68 
69 enum GRN_TOKEN_GET = grn_token_mode.GRN_TOKENIZE_GET;
70 enum GRN_TOKEN_ADD = grn_token_mode.GRN_TOKENIZE_ADD;
71 enum GRN_TOKEN_DEL = grn_token_mode.GRN_TOKENIZE_DELETE;
72 
73 /*
74  * grn_token_status is a flag set for tokenizer status codes.
75  * If a document or query contains no tokens, push an empty string with
76  * GRN_TOKEN_LAST as a token.
77  *
78  * @since 4.0.8
79  */
80 alias grn_token_status = uint;
81 
82 /*
83  * GRN_TOKEN_CONTINUE means that the next token is not the last one.
84  *
85  * @since 4.0.8
86  */
87 enum GRN_TOKEN_CONTINUE = 0;
88 /*
89  * GRN_TOKEN_LAST means that the next token is the last one.
90  *
91  * @since 4.0.8
92  */
93 enum GRN_TOKEN_LAST = 0x01L << 0;
94 /*
95  * GRN_TOKEN_OVERLAP means that ...
96  *
97  * @since 4.0.8
98  */
99 enum GRN_TOKEN_OVERLAP = 0x01L << 1;
100 /*
101  * GRN_TOKEN_UNMATURED means that ...
102  *
103  * @since 4.0.8
104  */
105 enum GRN_TOKEN_UNMATURED = 0x01L << 2;
106 /*
107  * GRN_TOKEN_REACH_END means that ...
108  *
109  * @since 4.0.8
110  */
111 enum GRN_TOKEN_REACH_END = 0x01L << 3;
112 /*
113  * GRN_TOKEN_SKIP means that the token is skipped
114  *
115  * @since 4.0.8
116  */
117 enum GRN_TOKEN_SKIP = 0x01L << 4;
118 /*
119  * GRN_TOKEN_SKIP_WITH_POSITION means that the token and postion is skipped
120  *
121  * @since 4.0.8
122  */
123 enum GRN_TOKEN_SKIP_WITH_POSITION = 0x01L << 5;
124 /*
125  * GRN_TOKEN_FORCE_PREFIX means that the token is used common prefix search
126  *
127  * @since 4.0.8
128  */
129 enum GRN_TOKEN_FORCE_PREFIX = 0x01L << 6;
130 
131 /*
132  * GRN_TOKEN_KEEP_ORIGINAL means that the original token of the token is
133  * also used. This is for token filter.
134  *
135  * @since 11.0.3
136  */
137 enum GRN_TOKEN_KEEP_ORIGINAL = 0x01L << 7;
138 
139 extern struct _grn_token;
140 alias grn_token = _grn_token;
141 
142 //GRN_API
143 groonga_d.groonga.grn_obj* grn_token_get_data(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
144 
145 //GRN_API
146 const (char)* grn_token_get_data_raw(groonga_d.groonga.grn_ctx* ctx, grn_token* token, size_t* length);
147 
148 //GRN_API
149 groonga_d.groonga.grn_rc grn_token_set_data(groonga_d.groonga.grn_ctx* ctx, grn_token* token, const (char)* str_ptr, int str_length);
150 
151 //GRN_API
152 grn_token_status grn_token_get_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
153 
154 //GRN_API
155 groonga_d.groonga.grn_rc grn_token_set_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status);
156 
157 //GRN_API
158 groonga_d.groonga.grn_rc grn_token_add_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status);
159 
160 //GRN_API
161 groonga_d.groonga.grn_rc grn_token_remove_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status);
162 
163 //GRN_API
164 ulong grn_token_get_source_offset(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
165 
166 //GRN_API
167 groonga_d.groonga.grn_rc grn_token_set_source_offset(groonga_d.groonga.grn_ctx* ctx, grn_token* token, ulong offset);
168 
169 //GRN_API
170 uint grn_token_get_source_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
171 
172 //GRN_API
173 groonga_d.groonga.grn_rc grn_token_set_source_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint length);
174 
175 //GRN_API
176 uint grn_token_get_source_first_character_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
177 
178 //GRN_API
179 groonga_d.groonga.grn_rc grn_token_set_source_first_character_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint length);
180 
181 //GRN_API
182 groonga_d.groonga.grn_bool grn_token_have_overlap(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
183 
184 //GRN_API
185 groonga_d.groonga.grn_rc grn_token_set_overlap(groonga_d.groonga.grn_ctx* ctx, grn_token* token, groonga_d.groonga.grn_bool have_overlap);
186 
187 //GRN_API
188 groonga_d.groonga.grn_obj* grn_token_get_metadata(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
189 
190 //GRN_API
191 groonga_d.groonga.grn_bool grn_token_get_force_prefix_search(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
192 
193 //GRN_API
194 groonga_d.groonga.grn_rc grn_token_set_force_prefix_search(groonga_d.groonga.grn_ctx* ctx, grn_token* token, groonga_d.groonga.grn_bool force);
195 
196 //GRN_API
197 uint grn_token_get_position(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
198 
199 //GRN_API
200 groonga_d.groonga.grn_rc grn_token_set_position(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint position);
201 
202 //GRN_API
203 float grn_token_get_weight(groonga_d.groonga.grn_ctx* ctx, grn_token* token);
204 
205 //GRN_API
206 groonga_d.groonga.grn_rc grn_token_set_weight(groonga_d.groonga.grn_ctx* ctx, grn_token* token, float weight);