|
1
|
|
|
2
3
|
#include <vector>
#include <iterator>
|
|
4
|
#include <algorithm>
|
|
5
|
#include <inttypes.h>
|
|
6
|
#include <iostream>
|
|
7
|
#include "deserialization/endianness.hpp"
|
|
8
|
#include "utf8.h"
|
|
9
|
#include "utf8sqlite.hpp"
|
|
10
|
#include "CharsetConverter.hpp"
|
|
11
|
#include "conversion_tables.hpp"
|
|
12
|
|
|
13
14
|
using namespace std;
|
|
15
16
|
namespace morfeusz {
|
|
17
|
const char DEFAULT_UNDEFINED_CHAR = static_cast<char> (0xF7);
|
|
18
|
|
|
19
20
|
string CharsetConverter::toString(const vector<uint32_t>& codepoints) const {
string res;
|
|
21
22
|
for (unsigned int i = 0; i < codepoints.size(); i++) {
this->append(codepoints[i], res);
|
|
23
24
25
26
|
}
return res;
}
|
|
27
|
CharsetConverter::~CharsetConverter() {
|
|
28
|
|
|
29
30
|
}
|
|
31
32
33
34
35
|
static inline void iterateThroughInvalidUtf8Sequence(const char*& it, const char* end) {
uint32_t _dupa;
while (it != end && utf8::internal::validate_next(it, end, _dupa) != utf8::internal::UTF8_OK) {
it++;
}
|
|
36
37
|
}
|
|
38
39
40
41
42
43
44
|
const UTF8CharsetConverter& UTF8CharsetConverter::getInstance() {
static UTF8CharsetConverter instance;
return instance;
}
UTF8CharsetConverter::UTF8CharsetConverter() {}
|
|
45
|
uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
|
|
46
|
return readUTF8((const unsigned char*&) it, (const unsigned char*) end);
|
|
47
|
}
|
|
48
|
|
|
49
|
void UTF8CharsetConverter::append(uint32_t cp, string& result) const {
|
|
50
|
utf8::unchecked::append(cp, back_inserter(result));
|
|
51
|
}
|
|
52
|
|
|
53
|
static vector<char> reverseArray(const uint32_t* array) {
|
|
54
55
56
57
|
vector<char> res;
unsigned char c = 0;
do {
uint32_t codepoint = array[c];
|
|
58
59
|
res.resize(max(static_cast<uint32_t> (res.size()), codepoint + 1), DEFAULT_UNDEFINED_CHAR);
res[codepoint] = static_cast<char> (c);
|
|
60
|
c++;
|
|
61
62
|
}
while (c != 255);
|
|
63
|
return res;
|
|
64
65
66
67
|
}
OneByteCharsetConverter::OneByteCharsetConverter(const uint32_t* array)
: array(array),
|
|
68
|
codepoint2Char(reverseArray(array)) {
|
|
69
70
|
}
|
|
71
|
// TODO - sprawdzanie zakresu
|
|
72
|
|
|
73
|
uint32_t OneByteCharsetConverter::next(const char*& it, const char* end) const {
|
|
74
|
return this->array[static_cast<unsigned char> (*it++)];
|
|
75
76
77
|
}
void OneByteCharsetConverter::append(uint32_t cp, std::string& result) const {
|
|
78
79
|
if (cp < this->codepoint2Char.size()) {
result.push_back(this->codepoint2Char[cp]);
|
|
80
81
|
}
else {
|
|
82
83
|
result.push_back(DEFAULT_UNDEFINED_CHAR);
}
|
|
84
85
|
}
|
|
86
87
88
89
90
|
const ISO8859_2_CharsetConverter& ISO8859_2_CharsetConverter::getInstance() {
static ISO8859_2_CharsetConverter instance;
return instance;
}
|
|
91
92
|
ISO8859_2_CharsetConverter::ISO8859_2_CharsetConverter()
: OneByteCharsetConverter(ISO_8859_2_TO_CODEPOINT) {
|
|
93
94
|
}
|
|
95
96
97
98
99
|
const Windows_1250_CharsetConverter& Windows_1250_CharsetConverter::getInstance() {
static Windows_1250_CharsetConverter instance;
return instance;
}
|
|
100
101
|
Windows_1250_CharsetConverter::Windows_1250_CharsetConverter()
: OneByteCharsetConverter(WINDOWS_1250_TO_CODEPOINT) {
|
|
102
|
}
|
|
103
|
|
|
104
105
106
107
108
|
const CP852_CharsetConverter& CP852_CharsetConverter::getInstance() {
static CP852_CharsetConverter instance;
return instance;
}
|
|
109
110
111
112
|
CP852_CharsetConverter::CP852_CharsetConverter()
: OneByteCharsetConverter(CP852_TO_CODEPOINT) {
}
|
|
113
114
115
116
117
118
119
120
121
122
|
string CharsetConverter::fromUTF8(const string& input) const {
string res;
const char* currInput = input.c_str();
const char* inputEnd = input.c_str() + input.length();
while (currInput != inputEnd) {
uint32_t cp = utf8::next(currInput, inputEnd);
this->append(cp, res);
}
return res;
}
|
|
123
|
|
|
124
125
126
127
128
129
130
131
132
133
134
|
string CharsetConverter::toUTF8(const string& input) const {
string res;
const char* currInput = input.c_str();
const char* inputEnd = input.c_str() + input.length();
while (currInput != inputEnd) {
uint32_t cp = this->next(currInput, inputEnd);
UTF8CharsetConverter::getInstance().append(cp, res);
}
return res;
}
|
|
135
136
137
|
string UTF8CharsetConverter::fromUTF8(const string& input) const {
return input;
}
|
|
138
|
|
|
139
140
141
142
|
string UTF8CharsetConverter::toUTF8(const string& input) const {
return input;
}
|
|
143
|
}
|