Skip to content

Commit 1731ba6

Browse files
lihuibaliulanzheng
lihuiba
authored and
liulanzheng
committed
make simple dom preseving the nodes' order of the document, by
creating and sorting an index, instead of the nodes themselves.
1 parent 2d27d09 commit 1731ba6

5 files changed

Lines changed: 76 additions & 50 deletions

File tree

‎common/stream.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@ IStream::ReadAll IStream::readall(size_t max_buf, size_t min_buf) {
2727
buf.size = -buf.size;
2828
LOG_ERROR_RETURN(ENOBUFS, buf, "content size in stream exceeds upper limit ", max_buf);
2929
}
30-
auto ptr = realloc(buf.ptr.get(), capacity *= 2);
30+
capacity *= 2;
31+
if ((size_t) capacity > max_buf) capacity = max_buf;
32+
auto ptr = realloc(buf.ptr.get(), capacity);
3133
if (!ptr) {
3234
buf.size = -buf.size;
3335
LOG_ERROR_RETURN(ENOBUFS, buf, "failed to realloc(`)", capacity);
3436
}
37+
buf.ptr.release();
3538
buf.ptr.reset(ptr);
3639
}
3740
}

‎ecosystem/simple_dom.cpp‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -102,16 +102,43 @@ class DocNode : public NodeImpl {
102102
LOG_DEBUG(VALUE(depth), k, ':', v);
103103
}
104104
}
105-
void set_children(vector<Derived>&& nodes, bool _sort = true) {
105+
struct Compare {
106+
const DocNode* _this;
107+
bool operator()(uint32_t i, uint32_t j) const {
108+
auto si = _this->_children[i].get_key();
109+
auto sj = _this->_children[j].get_key();
110+
return si < sj || (si == sj && i < j);
111+
}
112+
bool operator()(str s, uint32_t j) const {
113+
return s < _this->_children[j].get_key();
114+
}
115+
bool operator()(uint32_t i, str s) const {
116+
return _this->_children[i].get_key() < s;
117+
}
118+
};
119+
std::unique_ptr<uint32_t[]> _index;
120+
void set_children(vector<Derived>&& nodes, bool _indexing = true) {
106121
if (nodes.empty()) return;
107122
assert(nodes.size() <= MAX_NCHILDREN);
108123
if (nodes.size() > MAX_NCHILDREN)
109124
nodes.resize(MAX_NCHILDREN);
110-
if (_sort)
111-
sort(nodes.begin(), nodes.end());
112-
nodes.back()._flags |= FLAG_IS_LAST; // must be after sort!!!
113-
_nchildren = nodes.size();
114125
_children = std::move(nodes);
126+
_nchildren = _children.size();
127+
if (_indexing) {
128+
_index.reset(new uint32_t[_nchildren]);
129+
for (size_t i = 0; i < _nchildren; ++i) _index[i] = i;
130+
std::sort(_index.get(), _index.get() + _nchildren, Compare{this});
131+
Derived *a, *b = &_children[_index[0]];
132+
for (size_t i = 1; i < _nchildren; ++i) {
133+
a = b; b = &_children[_index[i]];
134+
if (a->get_key() != b->get_key())
135+
a->_flags |= FLAG_EQUAL_KEY_LAST;
136+
}
137+
b->_flags |= FLAG_EQUAL_KEY_LAST;
138+
}
139+
// user-side has no idea about # of children,
140+
// so we use this flag to indicate ending
141+
_children.back()._flags |= FLAG_IS_LAST;
115142
}
116143
~DocNode() override {
117144
if (is_root()) {
@@ -121,16 +148,20 @@ class DocNode : public NodeImpl {
121148
}
122149
}
123150
const NodeImpl* get(size_t i) const override {
124-
return (i < _children.size()) ? &_children[i] : nullptr;
151+
return (i < _nchildren) ? &_children[i] : nullptr;
125152
}
126153
const NodeImpl* get(str key) const override {
127154
if (_children.empty()) return nullptr;
128-
for (size_t i = 0; i < _children.size() - 1; ++i) {
155+
for (size_t i = 0; i < _nchildren - 1U; ++i) {
129156
assert((_children[i]._flags & FLAG_IS_LAST) == 0);
130157
}
131158
assert(_children.back()._flags & FLAG_IS_LAST);
132-
auto it = std::lower_bound(_children.begin(), _children.end(), key);
133-
return (it == _children.end() || it->get_key() != key) ? nullptr : &*it;
159+
if (!_index) return nullptr;
160+
auto end = _index.get() + _nchildren;
161+
auto it = std::lower_bound(_index.get(), end, key, Compare{this});
162+
if (it == end || *it >= _nchildren || key != _children[*it].get_key())
163+
return nullptr;
164+
return &_children[*it];
134165
}
135166
};
136167

@@ -196,13 +227,13 @@ struct JHandler : public BaseReaderHandler<UTF8<>, JHandler> {
196227
commit(true);
197228
return true;
198229
}
199-
void commit(bool sort) {
230+
void commit(bool _indexing) {
200231
assert(_nodes.size() > 1);
201232
auto temp = std::move(_nodes.back());
202233
_nodes.pop_back();
203234
assert(_nodes.back().size() > 0);
204-
// LOG_DEBUG(temp.size(), " elements to ", _nodes.back().back().get_key(), " sort=", sort);
205-
_nodes.back().back().set_children(std::move(temp), sort);
235+
// LOG_DEBUG(temp.size(), " elements to ", _nodes.back().back().get_key(), VALUE(_indexing));
236+
_nodes.back().back().set_children(std::move(temp), _indexing);
206237
}
207238
bool StartArray() {
208239
emplace_back(0, 0);
@@ -318,7 +349,9 @@ Node parse(char* text, size_t size, int flags) {
318349
if (flags & DOC_FREE_TEXT_IF_PARSING_FAILED) free(text);
319350
LOG_ERROR_RETURN(EINVAL, nullptr, "invalid document type ", HEX(i));
320351
}
321-
return parsers[i](text, size, flags);
352+
auto r = parsers[i](text, size, flags);
353+
if (!r && (flags & DOC_FREE_TEXT_IF_PARSING_FAILED)) free(text);
354+
return r;
322355
}
323356

324357
Node parse_file(fs::IFile* file, int flags) {

‎ecosystem/simple_dom.h‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ const int DOC_TYPE_MASK = 0xff;
140140

141141
const int DOC_FREE_TEXT_IF_PARSING_FAILED = 0x100;
142142
const int DOC_FREE_TEXT_ON_DESTRUCTION = 0x200;
143-
const int DOC_OWN_TEXT = 0x300;
143+
const int DOC_OWN_TEXT = DOC_FREE_TEXT_IF_PARSING_FAILED |
144+
DOC_FREE_TEXT_ON_DESTRUCTION;
144145

145146
using Document = Node;
146147

@@ -153,10 +154,12 @@ Node parse(char* text, size_t size, int flags);
153154
inline Node parse(IStream::ReadAll&& buf, int flags) {
154155
if (!buf.ptr || buf.size <= 0) return nullptr;
155156
auto node = parse((char*)buf.ptr.get(), (size_t)buf.size, flags);
156-
if (node || (flags & DOC_FREE_TEXT_IF_PARSING_FAILED)) {
157+
if (node) {
158+
buf.ptr.release();
159+
} else if (flags & DOC_FREE_TEXT_IF_PARSING_FAILED) {
157160
buf.ptr.reset();
158-
buf.size = 0;
159161
}
162+
buf.size = 0;
160163
return node;
161164
}
162165

@@ -196,26 +199,19 @@ inline auto Node::enumerable_children() const ->
196199
}
197200

198201
struct Node::SameKeyEnumerator : public Node::ChildrenEnumerator {
199-
const char* _base;
200-
str _key;
202+
const char* _base = nullptr;
201203
SameKeyEnumerator(const NodeImpl* node) {
202-
_impl = node;
203-
if (node) {
204+
if ((_impl = node)) {
204205
_base = node->get_root()->_text_begin;
205-
_key = node->get_key(_base);
206-
} else {
207-
_base = nullptr;
208-
assert(_key.empty());
209206
}
210207
}
211208
int next() {
212-
if (!valid()) return -1;
213-
_impl = _impl->next_sibling();
214-
if (!valid()) return -1;
215-
if (_impl->get_key(_base) != _key) {
216-
_impl = nullptr;
217-
return -1;
218-
}
209+
if (!valid() || (_impl->_flags & NodeImpl::FLAG_EQUAL_KEY_LAST)) return -1;
210+
auto key = _impl->get_key(_base);
211+
do {
212+
_impl = _impl->next_sibling();
213+
if (!valid()) return -1;
214+
} while (key != _impl->get_key(_base));
219215
return 0;
220216
}
221217
};

‎ecosystem/simple_dom_impl.h‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,10 @@ using str = estring_view;
3737
class NodeImpl : public Object {
3838
protected:
3939
NodeImpl() = default;
40-
const static uint8_t FLAG_IS_ROOT = 1;
41-
const static uint8_t FLAG_IS_LAST = 2;
42-
const static uint8_t FLAG_TEXT_OWNERSHIP= 4;
40+
const static uint8_t FLAG_IS_ROOT = 1; // the root node
41+
const static uint8_t FLAG_IS_LAST = 2; // last sibling node
42+
const static uint8_t FLAG_EQUAL_KEY_LAST= 4; // last node with same key
43+
const static uint8_t FLAG_TEXT_OWNERSHIP= 8;
4344
const static size_t MAX_NODE_SIZE = 256;
4445
const static size_t MAX_NCHILDREN = UINT16_MAX;
4546
const static size_t MAX_KEY_OFFSET = UINT32_MAX;
@@ -53,19 +54,17 @@ union { struct { // for non-root nodes
5354
uint16_t _k_len : 12; // key length (12 bits)
5455
uint16_t _v_off : 12; // value offset (12 bits) to key end
5556
}__attribute__((packed));
56-
uint32_t _k_off; // key offset to _text_begin
5757
const NodeImpl* _root; // root node
58-
uint32_t _v_len; // value length
59-
} ; // packed as 20 bytes
58+
}; // packed as 20 bytes
6059
struct { // for the root node
6160
uint8_t _flags_; // the same as _flags
6261
uint8_t _node_size; // sizeof(the node implementation)
6362
mutable uint16_t _refcnt; // reference counter of the document
64-
uint32_t _k_off_;
6563
const char* _text_begin;
66-
uint32_t _v_len_;
6764
}; };
68-
uint16_t _nchildren; // for all nodes
65+
uint32_t _k_off; // key offset to _text_begin
66+
uint32_t _v_len; // value length
67+
uint32_t _nchildren; // for all nodes
6968

7069
using AT16 = std::atomic<uint16_t>;
7170
static_assert(sizeof(AT16) == sizeof(_refcnt), "...");
@@ -107,10 +106,10 @@ struct { // for the root node
107106
return is_root() ? this : _root;
108107
}
109108
str get_key() const {
110-
return {get_root()->_text_begin + _k_off, _k_len};
109+
return get_key(get_root()->_text_begin);
111110
}
112111
str get_value() const {
113-
return {get_key().end() + _v_off, _v_len};
112+
return get_value(get_root()->_text_begin);
114113
}
115114
str get_key(const char* text_begin) const {
116115
return {text_begin + _k_off, _k_len};

‎ecosystem/test/test_simple_dom.cpp‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ const static char xml[] = R"(
5353
<DisplayName>1305433xxx</DisplayName>
5454
</Owner>
5555
</Contents>
56+
<asdf>asdf</asdf>
5657
<Contents>
5758
<Key>test100.txt</Key>
5859
<LastModified>2020-05-26T07:50:20.000Z</LastModified>
@@ -141,15 +142,9 @@ TEST(simple_dom, oss_list) {
141142
string marker;
142143
do_list_object("", list, &marker);
143144
static ObjectList truth = {
144-
{0, DT_REG, "test100.txt", 1, false},
145145
{0, DT_REG, "test10.txt", 1, false},
146+
{0, DT_REG, "test100.txt", 1, false},
146147
};
147-
using T = decltype(truth[0]);
148-
auto cmp = [](T& a, T& b) {
149-
return std::get<2>(a) < std::get<2>(b);
150-
};
151-
std::sort(truth.begin(), truth.end(), cmp);
152-
std::sort(list.begin(), list.end(), cmp);
153148
EXPECT_EQ(list, truth);
154149
EXPECT_EQ(marker, "test100.txt");
155150
}

0 commit comments

Comments
 (0)