Blame view

morfeusz/charset/CharsetConverter.cpp 3.79 KB
Michał Lenart authored
1
Michał Lenart authored
2
3
#include <vector>
#include <iterator>
Michał Lenart authored
4
#include <algorithm>
Michał Lenart authored
5
#include <inttypes.h>
Michał Lenart authored
6
#include <iostream>
Michał Lenart authored
7
#include "deserialization/endianness.hpp"
Michał Lenart authored
8
#include "utf8.h"
Michał Lenart authored
9
#include "utf8sqlite.hpp"
Michał Lenart authored
10
#include "CharsetConverter.hpp"
Michał Lenart authored
11
#include "conversion_tables.hpp"
Michał Lenart authored
12
Michał Lenart authored
13
14
using namespace std;
Michał Lenart authored
15
16
namespace morfeusz {
Michał Lenart authored
17
const char DEFAULT_UNDEFINED_CHAR = static_cast<char> (0xF7);
Michał Lenart authored
18
Michał Lenart authored
19
20
string CharsetConverter::toString(const vector<uint32_t>& codepoints) const {
    string res;
Michał Lenart authored
21
22
    for (unsigned int i = 0; i < codepoints.size(); i++) {
        this->append(codepoints[i], res);
Michał Lenart authored
23
24
25
26
    }
    return res;
}
Michał Lenart authored
27
CharsetConverter::~CharsetConverter() {
Michał Lenart authored
28
Michał Lenart authored
29
30
}
Michał Lenart authored
31
32
33
34
35
static inline void iterateThroughInvalidUtf8Sequence(const char*& it, const char* end) {
    uint32_t _dupa;
    while (it != end && utf8::internal::validate_next(it, end, _dupa) != utf8::internal::UTF8_OK) {
        it++;
    }
Michał Lenart authored
36
37
}
Michał Lenart authored
38
39
40
41
42
43
44
const UTF8CharsetConverter& UTF8CharsetConverter::getInstance() {
    static UTF8CharsetConverter instance;
    return instance;
}

UTF8CharsetConverter::UTF8CharsetConverter() {}
Michał Lenart authored
45
uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
Michał Lenart authored
46
    return readUTF8((const unsigned char*&) it, (const unsigned char*) end);
Michał Lenart authored
47
}
Michał Lenart authored
48
Michał Lenart authored
49
void UTF8CharsetConverter::append(uint32_t cp, string& result) const {
Michał Lenart authored
50
    utf8::unchecked::append(cp, back_inserter(result));
Michał Lenart authored
51
}
Michał Lenart authored
52
Michał Lenart authored
53
static vector<char> reverseArray(const uint32_t* array) {
Michał Lenart authored
54
55
56
57
    vector<char> res;
    unsigned char c = 0;
    do {
        uint32_t codepoint = array[c];
Michał Lenart authored
58
59
        res.resize(max(static_cast<uint32_t> (res.size()), codepoint + 1), DEFAULT_UNDEFINED_CHAR);
        res[codepoint] = static_cast<char> (c);
Michał Lenart authored
60
        c++;
Michał Lenart authored
61
62
    }
    while (c != 255);
Michał Lenart authored
63
    return res;
Michał Lenart authored
64
65
66
67
}

OneByteCharsetConverter::OneByteCharsetConverter(const uint32_t* array)
: array(array),
Michał Lenart authored
68
codepoint2Char(reverseArray(array)) {
Michał Lenart authored
69
70
}
Michał Lenart authored
71
// TODO - sprawdzanie zakresu
Michał Lenart authored
72
Michał Lenart authored
73
uint32_t OneByteCharsetConverter::next(const char*& it, const char* end) const {
Michał Lenart authored
74
    return this->array[static_cast<unsigned char> (*it++)];
Michał Lenart authored
75
76
77
}

void OneByteCharsetConverter::append(uint32_t cp, std::string& result) const {
Michał Lenart authored
78
79
    if (cp < this->codepoint2Char.size()) {
        result.push_back(this->codepoint2Char[cp]);
Michał Lenart authored
80
81
    }
    else {
Michał Lenart authored
82
83
        result.push_back(DEFAULT_UNDEFINED_CHAR);
    }
Michał Lenart authored
84
85
}
Michał Lenart authored
86
87
88
89
90
const ISO8859_2_CharsetConverter& ISO8859_2_CharsetConverter::getInstance() {
    static ISO8859_2_CharsetConverter instance;
    return instance;
}
Michał Lenart authored
91
92
ISO8859_2_CharsetConverter::ISO8859_2_CharsetConverter()
: OneByteCharsetConverter(ISO_8859_2_TO_CODEPOINT) {
Michał Lenart authored
93
94
}
Michał Lenart authored
95
96
97
98
99
const Windows_1250_CharsetConverter& Windows_1250_CharsetConverter::getInstance() {
    static Windows_1250_CharsetConverter instance;
    return instance;
}
Michał Lenart authored
100
101
Windows_1250_CharsetConverter::Windows_1250_CharsetConverter()
: OneByteCharsetConverter(WINDOWS_1250_TO_CODEPOINT) {
Michał Lenart authored
102
}
Michał Lenart authored
103
Michał Lenart authored
104
105
106
107
108
const CP852_CharsetConverter& CP852_CharsetConverter::getInstance() {
    static CP852_CharsetConverter instance;
    return instance;
}
Michał Lenart authored
109
110
111
112
CP852_CharsetConverter::CP852_CharsetConverter()
: OneByteCharsetConverter(CP852_TO_CODEPOINT) {
}
Michał Lenart authored
113
114
115
116
117
118
119
120
121
122
string CharsetConverter::fromUTF8(const string& input) const {
    string res;
    const char* currInput = input.c_str();
    const char* inputEnd = input.c_str() + input.length();
    while (currInput != inputEnd) {
        uint32_t cp = utf8::next(currInput, inputEnd);
        this->append(cp, res);
    }
    return res;
}
Michał Lenart authored
123
Michał Lenart authored
124
125
126
127
128
129
130
131
132
133
134
string CharsetConverter::toUTF8(const string& input) const {
    string res;
    const char* currInput = input.c_str();
    const char* inputEnd = input.c_str() + input.length();
    while (currInput != inputEnd) {
        uint32_t cp = this->next(currInput, inputEnd);
        UTF8CharsetConverter::getInstance().append(cp, res);
    }
    return res;
}
Michał Lenart authored
135
136
137
string UTF8CharsetConverter::fromUTF8(const string& input) const {
    return input;
}
Michał Lenart authored
138
Michał Lenart authored
139
140
141
142
string UTF8CharsetConverter::toUTF8(const string& input) const {
    return input;
}
Michał Lenart authored
143
}