1 /* Copyright (c) 2008 Sascha Kohlmann
2 *
3 * This program is free software: you can redistribute it and/or modify
4 * it under the terms of the GNU Affero General Public License as published by
5 * the Free Software Foundation, either version 3 of the License, or
6 * (at your option) any later version.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU Affero General Public License for more details.
12 *
13 * You should have received a copy of the GNU Affero General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16 package net.sf.eos.analyzer;
17
18 /**
19 * An implementation splits text data into its {@link Token}.
20 * @author Sascha Kohlmann
21 */
22 public interface Tokenizer /*extends Configurable*/ {
23
24 /**
25 * The next token or {@code null}.
26 * @return the next token or {@code null}
27 */
28 Token next() throws TokenizerException;
29 }