Blame view

morfeusz/charset/CharsetConverter.cpp 2.89 KB
Michał Lenart authored
1
Michał Lenart authored
2
3
#include <vector>
#include <iterator>
Michał Lenart authored
4
#include <algorithm>
Michał Lenart authored
5
#include <inttypes.h>
Michał Lenart authored
6
#include <iostream>
Michał Lenart authored
7
#include "../endianness.hpp"
Michał Lenart authored
8
9
#include "utf8.h"
#include "CharsetConverter.hpp"
Michał Lenart authored
10
#include "conversion_tables.hpp"
Michał Lenart authored
11
Michał Lenart authored
12
13
using namespace std;
Michał Lenart authored
14
15
const char DEFAULT_UNDEFINED_CHAR = static_cast<char>(0xF7);
Michał Lenart authored
16
17
string CharsetConverter::toString(const vector<uint32_t>& codepoints) const {
    string res;
Michał Lenart authored
18
19
    for (unsigned int i = 0; i < codepoints.size(); i++) {
        this->append(codepoints[i], res);
Michał Lenart authored
20
21
22
23
    }
    return res;
}
Michał Lenart authored
24
25
26
27
CharsetConverter::~CharsetConverter() {

}
Michał Lenart authored
28
uint32_t UTF8CharsetConverter::peek(const char*& it, const char* end) const {
Michał Lenart authored
29
    return utf8::unchecked::peek_next(it);
Michał Lenart authored
30
31
}
Michał Lenart authored
32
uint32_t UTF8CharsetConverter::next(const char*& it, const char* end) const {
Michał Lenart authored
33
34
35
36
37
38
39
40
    return utf8::unchecked::next(it);
//    catch (utf8::exception ex) {
//        cerr << "WARNING: Replacing invalid character: " << hex << (uint16_t) *it << dec << " with replacement char: 0xFFFD" << endl;
//        while (it != end && peek(it, end) == 0xFFFD) {
//            utf8::unchecked::next(it);
//        }
//        return 0xFFFD;
//    }
Michał Lenart authored
41
}
Michał Lenart authored
42
Michał Lenart authored
43
void UTF8CharsetConverter::append(uint32_t cp, string& result) const {
Michał Lenart authored
44
    utf8::append(cp, back_inserter(result));
Michał Lenart authored
45
}
Michał Lenart authored
46
Michał Lenart authored
47
static vector<char> reverseArray(const uint32_t* array) {
Michał Lenart authored
48
49
50
51
    vector<char> res;
    unsigned char c = 0;
    do {
        uint32_t codepoint = array[c];
Michał Lenart authored
52
53
        res.resize(max(static_cast<uint32_t> (res.size()), codepoint + 1), DEFAULT_UNDEFINED_CHAR);
        res[codepoint] = static_cast<char> (c);
Michał Lenart authored
54
        c++;
Michał Lenart authored
55
    }    while (c != 255);
Michał Lenart authored
56
    return res;
Michał Lenart authored
57
58
59
60
}

OneByteCharsetConverter::OneByteCharsetConverter(const uint32_t* array)
: array(array),
Michał Lenart authored
61
codepoint2Char(reverseArray(array)) {
Michał Lenart authored
62
63
}
Michał Lenart authored
64
// TODO - sprawdzanie zakresu
Michał Lenart authored
65
Michał Lenart authored
66
uint32_t OneByteCharsetConverter::peek(const char*& it, const char* end) const {
Michał Lenart authored
67
    return this->array[static_cast<unsigned char> (*it)];
Michał Lenart authored
68
69
70
}

uint32_t OneByteCharsetConverter::next(const char*& it, const char* end) const {
Michał Lenart authored
71
    return this->array[static_cast<unsigned char> (*(it++))];
Michał Lenart authored
72
73
74
}

void OneByteCharsetConverter::append(uint32_t cp, std::string& result) const {
Michał Lenart authored
75
76
    if (cp < this->codepoint2Char.size()) {
        result.push_back(this->codepoint2Char[cp]);
Michał Lenart authored
77
    } else {
Michał Lenart authored
78
79
        result.push_back(DEFAULT_UNDEFINED_CHAR);
    }
Michał Lenart authored
80
81
}
Michał Lenart authored
82
83
ISO8859_2_CharsetConverter::ISO8859_2_CharsetConverter()
: OneByteCharsetConverter(ISO_8859_2_TO_CODEPOINT) {
Michał Lenart authored
84
85
}
Michał Lenart authored
86
87
Windows_1250_CharsetConverter::Windows_1250_CharsetConverter()
: OneByteCharsetConverter(WINDOWS_1250_TO_CODEPOINT) {
Michał Lenart authored
88
}
Michał Lenart authored
89
Michał Lenart authored
90
91
92
93
CP852_CharsetConverter::CP852_CharsetConverter()
: OneByteCharsetConverter(CP852_TO_CODEPOINT) {
}
Michał Lenart authored
94
95
96
97
98
99
100
101
102
103
string CharsetConverter::fromUTF8(const string& input) const {
    string res;
    const char* currInput = input.c_str();
    const char* inputEnd = input.c_str() + input.length();
    while (currInput != inputEnd) {
        uint32_t cp = utf8::next(currInput, inputEnd);
        this->append(cp, res);
    }
    return res;
}