TextReader.cpp
2.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/*
* File: TextReader.cpp
* Author: lennyn
*
* Created on May 28, 2014, 11:43 AM
*/
#include "TextReader.hpp"
#include "charset_utils.hpp"
TextReader::TextReader(
const char* inputStart,
const char* inputEnd,
const Environment& env)
: codepointsNum(0),
wordStartPtr(inputStart),
currPtr(inputStart),
inputEnd(inputEnd),
env(env),
knowsAboutWhitespace(false),
atWhitespace(false),
peekIsRead(false),
thePeek(0x00),
theNormalizedPeek(0x00),
ptrAfterThePeek(NULL) {}
void TextReader::markWordStartsHere() {
codepointsNum = 0;
wordStartPtr = currPtr;
}
const char* TextReader::getWordStartPtr() const {
return wordStartPtr;
}
const char* TextReader::getCurrPtr() const {
return currPtr;
}
const char* TextReader::getNextPtr() {
if (!peekIsRead) {
peek();
}
return ptrAfterThePeek;
}
const char* TextReader::getEndPtr() const {
return inputEnd;
}
int TextReader::getCodepointsRead() const {
return codepointsNum;
}
bool TextReader::isAtEnd() const {
return currPtr == inputEnd;
}
bool TextReader::isAtWhitespace() {
if (isAtEnd()) {
return true;
}
else {
if (!peekIsRead) {
peek();
}
return atWhitespace;
}
}
uint32_t TextReader::peek() {
if (peekIsRead) {
return thePeek;
}
else {
ptrAfterThePeek = currPtr;
thePeek = env.getCharsetConverter().next(ptrAfterThePeek, inputEnd);
theNormalizedPeek = env.getProcessorType() == ANALYZER
? env.getCaseConverter().toLower(thePeek)
: thePeek;
atWhitespace = isWhitespace(thePeek);
peekIsRead = true;
return thePeek;
}
}
uint32_t TextReader::normalizedPeek() {
if (!peekIsRead) {
peek();
}
return theNormalizedPeek;
}
uint32_t TextReader::next() {
if (!peekIsRead) {
peek();
}
currPtr = ptrAfterThePeek;
peekIsRead = false;
knowsAboutWhitespace = false;
codepointsNum++;
return thePeek;
}
void TextReader::skipWhitespaces() {
while (!isAtEnd() && isAtWhitespace()) {
next();
}
}
void TextReader::proceedToEnd() {
while (!isAtEnd()) {
next();
}
}
TextReader::~TextReader() {
}