Blame view

morfeusz/charset/charset_utils.hpp 903 Bytes
Michał Lenart authored
1
2
3
4
5
6
7
8
9
10
/* 
 * File:   charset_utils.hpp
 * Author: lennyn
 *
 * Created on November 15, 2013, 1:57 PM
 */

#ifndef CHARSET_UTILS_HPP
#define	CHARSET_UTILS_HPP
Michał Lenart authored
11
#include <string>
Michał Lenart authored
12
#include <set>
Michał Lenart authored
13
#include "CharsetConverter.hpp"
Michał Lenart authored
14
Michał Lenart authored
15
16
17
18
19
static inline std::set<int> initializeWhitespaces() {
    std::set<int> res;
    res.insert(0x00);
    res.insert(0x0A);
    res.insert(0x20);
Michał Lenart authored
20
    res.insert('\t');
Michał Lenart authored
21
22
23
    return res;
}
Michał Lenart authored
24
inline bool isEndOfWord(int codepoint) {
Michał Lenart authored
25
    static std::set<int> whitespaces(initializeWhitespaces());
Michał Lenart authored
26
    return whitespaces.count(codepoint);
Michał Lenart authored
27
}
Michał Lenart authored
28
Michał Lenart authored
29
30
31
32
33
34
35
36
37
38
39
40
template <class StateClass>
void feedState(
        StateClass& state,
        int codepoint,
        const CharsetConverter& charsetConverter) {
    std::string chars;
    charsetConverter.append(codepoint, chars);
    for (unsigned int i = 0; i < chars.length(); i++) {
        state.proceedToNext(chars[i]);
    }
}
Michał Lenart authored
41
42
#endif	/* CHARSET_UTILS_HPP */