1 /* -*- c-basic-offset: 2 -*- */ 2 /* 3 Copyright(C) 2014-2018 Brazil 4 Copyright(C) 2018-2021 Sutou Kouhei <kou@clear-code.com> 5 6 This library is free software; you can redistribute it and/or 7 modify it under the terms of the GNU Lesser General Public 8 License version 2.1 as published by the Free Software Foundation. 9 10 This library is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 Lesser General Public License for more details. 14 15 You should have received a copy of the GNU Lesser General Public 16 License along with this library; if not, write to the Free Software 17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 */ 19 module groonga_d.token; 20 21 22 private static import groonga_d.groonga; 23 24 extern(C): 25 nothrow @nogc: 26 27 /* 28 * grn_tokenize_mode describes propose for tokenization. 29 * 30 * `GRN_TOKENIZE_GET`: Tokenize for search. 31 * 32 * `GRN_TOKENIZE_ADD`: Tokenize for adding token to index. 33 * 34 * `GRN_TOKENIZE_DELETE`: Tokenize for deleting token from index. 35 * 36 * @since 4.0.8 37 */ 38 enum grn_tokenize_mode 39 { 40 GRN_TOKENIZE_GET = 0, 41 GRN_TOKENIZE_ADD, 42 GRN_TOKENIZE_DELETE, 43 GRN_TOKENIZE_ONLY, 44 } 45 46 //Declaration name in C language 47 enum 48 { 49 GRN_TOKENIZE_GET = .grn_tokenize_mode.GRN_TOKENIZE_GET, 50 GRN_TOKENIZE_ADD = .grn_tokenize_mode.GRN_TOKENIZE_ADD, 51 GRN_TOKENIZE_DELETE = .grn_tokenize_mode.GRN_TOKENIZE_DELETE, 52 GRN_TOKENIZE_ONLY = .grn_tokenize_mode.GRN_TOKENIZE_ONLY, 53 } 54 55 /* 56 grn_token_mode describes propose for tokenization. 57 58 `GRN_TOKEN_GET`: Tokenization for search. 59 60 `GRN_TOKEN_ADD`: Tokenization for adding token to index. 61 62 `GRN_TOKEN_DEL`: Tokenization for deleting token from index. 63 64 @since 4.0.7 65 @deprecated since 4.0.8. Use grn_tokenize_mode instead. 66 */ 67 alias grn_token_mode = grn_tokenize_mode; 68 69 enum GRN_TOKEN_GET = grn_token_mode.GRN_TOKENIZE_GET; 70 enum GRN_TOKEN_ADD = grn_token_mode.GRN_TOKENIZE_ADD; 71 enum GRN_TOKEN_DEL = grn_token_mode.GRN_TOKENIZE_DELETE; 72 73 /* 74 * grn_token_status is a flag set for tokenizer status codes. 75 * If a document or query contains no tokens, push an empty string with 76 * GRN_TOKEN_LAST as a token. 77 * 78 * @since 4.0.8 79 */ 80 alias grn_token_status = uint; 81 82 /* 83 * GRN_TOKEN_CONTINUE means that the next token is not the last one. 84 * 85 * @since 4.0.8 86 */ 87 enum GRN_TOKEN_CONTINUE = 0; 88 /* 89 * GRN_TOKEN_LAST means that the next token is the last one. 90 * 91 * @since 4.0.8 92 */ 93 enum GRN_TOKEN_LAST = 0x01L << 0; 94 /* 95 * GRN_TOKEN_OVERLAP means that ... 96 * 97 * @since 4.0.8 98 */ 99 enum GRN_TOKEN_OVERLAP = 0x01L << 1; 100 /* 101 * GRN_TOKEN_UNMATURED means that ... 102 * 103 * @since 4.0.8 104 */ 105 enum GRN_TOKEN_UNMATURED = 0x01L << 2; 106 /* 107 * GRN_TOKEN_REACH_END means that ... 108 * 109 * @since 4.0.8 110 */ 111 enum GRN_TOKEN_REACH_END = 0x01L << 3; 112 /* 113 * GRN_TOKEN_SKIP means that the token is skipped 114 * 115 * @since 4.0.8 116 */ 117 enum GRN_TOKEN_SKIP = 0x01L << 4; 118 /* 119 * GRN_TOKEN_SKIP_WITH_POSITION means that the token and postion is skipped 120 * 121 * @since 4.0.8 122 */ 123 enum GRN_TOKEN_SKIP_WITH_POSITION = 0x01L << 5; 124 /* 125 * GRN_TOKEN_FORCE_PREFIX means that the token is used common prefix search 126 * 127 * @since 4.0.8 128 */ 129 enum GRN_TOKEN_FORCE_PREFIX = 0x01L << 6; 130 131 /* 132 * GRN_TOKEN_KEEP_ORIGINAL means that the original token of the token is 133 * also used. This is for token filter. 134 * 135 * @since 11.0.3 136 */ 137 enum GRN_TOKEN_KEEP_ORIGINAL = 0x01L << 7; 138 139 extern struct _grn_token; 140 alias grn_token = _grn_token; 141 142 //GRN_API 143 groonga_d.groonga.grn_obj* grn_token_get_data(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 144 145 //GRN_API 146 const (char)* grn_token_get_data_raw(groonga_d.groonga.grn_ctx* ctx, grn_token* token, size_t* length); 147 148 //GRN_API 149 groonga_d.groonga.grn_rc grn_token_set_data(groonga_d.groonga.grn_ctx* ctx, grn_token* token, const (char)* str_ptr, int str_length); 150 151 //GRN_API 152 grn_token_status grn_token_get_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 153 154 //GRN_API 155 groonga_d.groonga.grn_rc grn_token_set_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status); 156 157 //GRN_API 158 groonga_d.groonga.grn_rc grn_token_add_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status); 159 160 //GRN_API 161 groonga_d.groonga.grn_rc grn_token_remove_status(groonga_d.groonga.grn_ctx* ctx, grn_token* token, grn_token_status status); 162 163 //GRN_API 164 ulong grn_token_get_source_offset(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 165 166 //GRN_API 167 groonga_d.groonga.grn_rc grn_token_set_source_offset(groonga_d.groonga.grn_ctx* ctx, grn_token* token, ulong offset); 168 169 //GRN_API 170 uint grn_token_get_source_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 171 172 //GRN_API 173 groonga_d.groonga.grn_rc grn_token_set_source_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint length); 174 175 //GRN_API 176 uint grn_token_get_source_first_character_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 177 178 //GRN_API 179 groonga_d.groonga.grn_rc grn_token_set_source_first_character_length(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint length); 180 181 //GRN_API 182 groonga_d.groonga.grn_bool grn_token_have_overlap(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 183 184 //GRN_API 185 groonga_d.groonga.grn_rc grn_token_set_overlap(groonga_d.groonga.grn_ctx* ctx, grn_token* token, groonga_d.groonga.grn_bool have_overlap); 186 187 //GRN_API 188 groonga_d.groonga.grn_obj* grn_token_get_metadata(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 189 190 //GRN_API 191 groonga_d.groonga.grn_bool grn_token_get_force_prefix_search(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 192 193 //GRN_API 194 groonga_d.groonga.grn_rc grn_token_set_force_prefix_search(groonga_d.groonga.grn_ctx* ctx, grn_token* token, groonga_d.groonga.grn_bool force); 195 196 //GRN_API 197 uint grn_token_get_position(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 198 199 //GRN_API 200 groonga_d.groonga.grn_rc grn_token_set_position(groonga_d.groonga.grn_ctx* ctx, grn_token* token, uint position); 201 202 //GRN_API 203 float grn_token_get_weight(groonga_d.groonga.grn_ctx* ctx, grn_token* token); 204 205 //GRN_API 206 groonga_d.groonga.grn_rc grn_token_set_weight(groonga_d.groonga.grn_ctx* ctx, grn_token* token, float weight);