Skip to content

Commit 1e4ba05

Browse files
author
Benjamin Forster
committed
refactor to enable configuration of index and prefixes
1 parent c9de920 commit 1e4ba05

7 files changed

Lines changed: 224 additions & 117 deletions

File tree

‎index.js‎

Lines changed: 72 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const inherits = require('inherits')
55
const events = require('events')
66
const SparqlIterator = require('sparql-iterator')
77

8+
const constants = require('./lib/constants')
89
const utils = require('./lib/utils')
910
const prefixes = require('./lib/prefixes')
1011
const Variable = require('./lib/Variable')
@@ -26,21 +27,29 @@ function Graph (storage, key, opts) {
2627
events.EventEmitter.call(this)
2728
this.db = hyperdb(storage, key, opts)
2829

30+
// what are the default prefixes
31+
this._prefixes = constants.DEFAULT_PREFIXES
32+
this._indexes = (opts && opts.index === 'small')
33+
? constants.HEXSTORE_INDEXES_REDUCED
34+
: constants.HEXSTORE_INDEXES
35+
this._indexKeys = Object.keys(this._indexes)
36+
2937
this.db.on('error', (e) => {
3038
this.emit('error', e)
3139
})
3240
this.db.on('ready', (e) => {
3341
this.emit('ready', e)
42+
// check prefixes
3443
})
3544
}
3645

3746
inherits(Graph, events.EventEmitter)
3847

3948
Graph.prototype.v = (name) => new Variable(name)
4049

41-
Graph.prototype.prefixes = function (callback) {
50+
Graph.prototype.listPrefixes = function (callback) {
4251
// should cache this somehow
43-
const prefixStream = this.db.createReadStream(prefixes.PREFIX_KEY)
52+
const prefixStream = this.db.createReadStream(constants.PREFIX_KEY)
4453
utils.collect(prefixStream, (err, data) => {
4554
if (err) return callback(err)
4655
var names = data.reduce((p, nodes) => {
@@ -57,8 +66,8 @@ Graph.prototype.addPrefix = function (prefix, uri, cb) {
5766
}
5867

5968
Graph.prototype.getStream = function (triple, opts) {
60-
const stream = this.db.createReadStream(utils.createQuery(triple))
61-
return stream.pipe(new HyperdbReadTransform(this.db, opts))
69+
const stream = this.db.createReadStream(this._createQuery(triple))
70+
return stream.pipe(new HyperdbReadTransform(this.db, this._prefixes, opts))
6271
}
6372

6473
Graph.prototype.get = function (triple, opts, callback) {
@@ -71,21 +80,22 @@ function doAction (action) {
7180
if (!triples) return callback(new Error('Must pass triple'))
7281
let entries = (!triples.reduce) ? [triples] : triples
7382
entries = entries.reduce((prev, triple) => {
74-
return prev.concat(this.generateBatch(triple, action))
83+
return prev.concat(this._generateBatch(triple, action))
7584
}, [])
7685
this.db.batch(entries.reverse(), callback)
7786
}
7887
}
7988

8089
function doActionStream (action) {
8190
return function () {
91+
const self = this
8292
const transform = new Transform({
8393
objectMode: true,
8494
transform (triples, encoding, done) {
8595
if (!triples) return done()
8696
let entries = (!triples.reduce) ? [triples] : triples
8797
entries = entries.reduce((prev, triple) => {
88-
return prev.concat(utils.generateBatch(triple, action))
98+
return prev.concat(self._generateBatch(triple, action))
8999
}, [])
90100
this.push(entries.reverse())
91101
done()
@@ -150,10 +160,64 @@ Graph.prototype.query = function (query, callback) {
150160
utils.collect(this.queryStream(query), callback)
151161
}
152162

153-
Graph.prototype.generateBatch = utils.generateBatch
154-
155163
Graph.prototype.close = function (callback) {
156164
callback()
157165
}
158166

167+
/* PRIVATE FUNCTIONS */
168+
169+
Graph.prototype._generateBatch = function (triple, action) {
170+
if (!action) action = 'put'
171+
var data = null
172+
if (action === 'put') {
173+
data = JSON.stringify(utils.extraDataMask(triple))
174+
}
175+
return this._encodeKeys(triple).map(key => ({
176+
type: 'put', // no delete in hyperdb so just putting nulls
177+
key: key,
178+
value: data
179+
}))
180+
}
181+
182+
Graph.prototype._encodeKeys = function (triple) {
183+
const encodedTriple = utils.encodeTriple(triple, this._prefixes)
184+
return this._indexKeys.map(key => utils.encodeKey(key, encodedTriple))
185+
}
186+
187+
Graph.prototype._createQuery = function (pattern, options) {
188+
var types = utils.typesFromPattern(pattern)
189+
var preferedIndex = options && options.index
190+
var index = this._findIndex(types, preferedIndex)
191+
const encodedTriple = utils.encodeTriple(pattern, this._prefixes)
192+
var key = utils.encodeKey(index, encodedTriple)
193+
return key
194+
}
195+
196+
Graph.prototype._possibleIndexes = function (types) {
197+
var result = this._indexKeys.filter((key) => {
198+
var matches = 0
199+
return this._indexes[key].every(function (e, i) {
200+
if (types.indexOf(e) >= 0) {
201+
matches++
202+
return true
203+
}
204+
if (matches === types.length) {
205+
return true
206+
}
207+
})
208+
})
209+
210+
result.sort()
211+
212+
return result
213+
}
214+
215+
Graph.prototype._findIndex = function (types, preferedIndex) {
216+
var result = this._possibleIndexes(types)
217+
if (preferedIndex && result.some(r => r === preferedIndex)) {
218+
return preferedIndex
219+
}
220+
return result[0]
221+
}
222+
159223
module.exports = Graph

‎lib/HyperdbReadTransform.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@ const Transform = require('readable-stream').Transform
22
const inherits = require('inherits')
33
const utils = require('./utils')
44

5-
function HyperdbReadTransform (db, options) {
5+
function HyperdbReadTransform (db, prefixes, options) {
66
if (!(this instanceof HyperdbReadTransform)) {
77
return new HyperdbReadTransform(db, options)
88
}
99
var opts = options || {}
1010
this.db = db
11+
this._prefixes = prefixes
1112
this._finished = false
1213
this._count = 0
1314
this._filter = opts.filter
@@ -33,7 +34,7 @@ HyperdbReadTransform.prototype._transform = function transform (nodes, encoding,
3334
}
3435
var value = nodes[0].value && JSON.parse(nodes[0].value.toString())
3536
if (value === null) return done()
36-
value = Object.assign(value, utils.decodeKey(nodes[0].key))
37+
value = Object.assign(value, utils.decodeKey(nodes[0].key, this._prefixes))
3738
if (!this._filter || this._filter(value)) {
3839
if (this._count >= this._offset) {
3940
this.push(value)

‎lib/constants.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
const HEXSTORE_INDEXES = {
2+
spo: ['subject', 'predicate', 'object'],
3+
pos: ['predicate', 'object', 'subject'],
4+
osp: ['object', 'subject', 'predicate'],
5+
sop: ['subject', 'object', 'predicate'], // [optional]
6+
pso: ['predicate', 'subject', 'object'], // [optional]
7+
ops: ['object', 'predicate', 'subject'] // [optional]
8+
}
9+
const HEXSTORE_INDEXES_REDUCED = {
10+
spo: HEXSTORE_INDEXES.spo,
11+
pos: HEXSTORE_INDEXES.pos,
12+
osp: HEXSTORE_INDEXES.osp
13+
}
14+
const PREFIX_KEY = '@prefix/'
15+
const DEFAULT_PREFIXES = {
16+
_: 'hg://',
17+
foaf: 'http://xmlns.com/foaf/0.1/'
18+
}
19+
20+
module.exports = {
21+
PREFIX_KEY,
22+
DEFAULT_PREFIXES,
23+
HEXSTORE_INDEXES,
24+
HEXSTORE_INDEXES_REDUCED
25+
}

‎lib/prefixes.js‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
const PREFIX_KEY = '@prefix/'
1+
var constants = require('./constants')
2+
3+
const PREFIX_KEY = constants.PREFIX_KEY
24
const PREFIX_REGEX = /^(\w+):/
3-
const DEFAULT_PREFIXES = {
4-
_: 'hg://',
5-
foaf: 'http://xmlns.com/foaf/0.1/'
6-
}
75

86
function toKey (prefix) {
97
return PREFIX_KEY + prefix
@@ -36,8 +34,6 @@ function fromPrefixed (uri, prefixes) {
3634
}
3735

3836
module.exports = {
39-
DEFAULT_PREFIXES,
40-
PREFIX_KEY,
4137
toKey,
4238
fromKey,
4339
fromNodes,

‎lib/utils.js‎

Lines changed: 30 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,13 @@
11
const Variable = require('./Variable')
22
const prefixes = require('./prefixes')
3+
const constants = require('./constants')
34

4-
const defs = {
5-
spo: ['subject', 'predicate', 'object'],
6-
sop: ['subject', 'object', 'predicate'], // [optional]
7-
pos: ['predicate', 'object', 'subject'],
8-
pso: ['predicate', 'subject', 'object'], // [optional]
9-
ops: ['object', 'predicate', 'subject'], // [optional]
10-
osp: ['object', 'subject', 'predicate']
11-
}
5+
const spo = constants.HEXSTORE_INDEXES.spo
126
const tripleAliasMap = {
137
s: 'subject',
148
p: 'predicate',
159
o: 'object'
1610
}
17-
const defKeys = Object.keys(defs)
1811

1912
function collect (stream, cb) {
2013
var res = []
@@ -25,45 +18,43 @@ function collect (stream, cb) {
2518

2619
function filterTriple (triple) {
2720
const filtered = {}
28-
defs.spo.forEach((key) => {
21+
spo.forEach((key) => {
2922
if (triple.hasOwnProperty(key)) {
3023
filtered[key] = triple[key]
3124
}
3225
})
3326
return filtered
3427
}
3528

36-
function escapeKeyValue (value) {
29+
function escapeKeyValue (value, prefixMap) {
3730
if (typeof value === 'string' || value instanceof String) {
38-
return prefixes.toPrefixed(value, prefixes.DEFAULT_PREFIXES).replace(/(\/)/g, '%2F')
31+
return prefixes.toPrefixed(value, prefixMap).replace(/(\/)/g, '%2F')
3932
}
4033
return value
4134
}
4235

43-
function unescapeKeyValue (value) {
44-
return prefixes.fromPrefixed(value.replace(/%2F/g, '/'), prefixes.DEFAULT_PREFIXES)
36+
function unescapeKeyValue (value, prefixMap) {
37+
return prefixes.fromPrefixed(value.replace(/%2F/g, '/'), prefixMap)
4538
}
4639

47-
function decodeKey (key) {
48-
const values = key.split('/')
49-
if (values.length < 4) throw new Error('Key is not in triple form')
50-
const order = values[0]
51-
const triple = {}
52-
for (var i = 0; i < 3; i++) {
53-
const k = tripleAliasMap[order[i]]
54-
triple[k] = unescapeKeyValue(values[i + 1])
55-
}
40+
function encodeTriple (triple, prefixMap) {
41+
// var newTriple = Object.assign({}, triple)
42+
spo.forEach((key) => {
43+
if (triple.hasOwnProperty(key)) {
44+
triple[key] = escapeKeyValue(triple[key], prefixMap)
45+
}
46+
})
5647
return triple
5748
}
5849

5950
function encodeKey (key, triple) {
6051
var result = key
61-
var def = defs[key]
52+
var def = constants.HEXSTORE_INDEXES[key]
6253
var i = 0
6354
var value = triple[def[i]]
6455
// need to handle this smarter
6556
while (value) {
66-
result += '/' + escapeKeyValue(value)
57+
result += '/' + value
6758
i += 1
6859
value = triple[def[i]]
6960
}
@@ -73,48 +64,16 @@ function encodeKey (key, triple) {
7364
return result
7465
}
7566

76-
function encodeKeys (triple) {
77-
return defKeys.map(key => encodeKey(key, triple))
78-
}
79-
80-
function generateBatch (triple, action) {
81-
if (!action) action = 'put'
82-
var data = null
83-
if (action === 'put') {
84-
data = JSON.stringify(extraDataMask(triple))
85-
}
86-
return encodeKeys(triple).map(key => ({
87-
type: 'put', // no delete in hyperdb so just putting nulls
88-
key: key,
89-
value: data
90-
}))
91-
}
92-
93-
function possibleIndexes (types) {
94-
var result = defKeys.filter((key) => {
95-
var matches = 0
96-
return defs[key].every(function (e, i) {
97-
if (types.indexOf(e) >= 0) {
98-
matches++
99-
return true
100-
}
101-
if (matches === types.length) {
102-
return true
103-
}
104-
})
105-
})
106-
107-
result.sort()
108-
109-
return result
110-
}
111-
112-
function findIndex (types, preferedIndex) {
113-
var result = possibleIndexes(types)
114-
if (preferedIndex && result.some(r => r === preferedIndex)) {
115-
return preferedIndex
67+
function decodeKey (key, prefixMap) {
68+
const values = key.split('/')
69+
if (values.length < 4) throw new Error('Key is not in triple form')
70+
const order = values[0]
71+
const triple = {}
72+
for (var i = 0; i < 3; i++) {
73+
const k = tripleAliasMap[order[i]]
74+
triple[k] = unescapeKeyValue(values[i + 1], prefixMap)
11675
}
117-
return result[0]
76+
return triple
11877
}
11978

12079
function typesFromPattern (pattern) {
@@ -132,16 +91,8 @@ function typesFromPattern (pattern) {
13291
})
13392
}
13493

135-
function createQuery (pattern, options) {
136-
var types = typesFromPattern(pattern)
137-
var preferedIndex = options && options.index
138-
var index = findIndex(types, preferedIndex)
139-
var key = encodeKey(index, pattern)
140-
return key
141-
}
142-
14394
function hasKey (key) {
144-
return defs.spo.indexOf(key) >= 0
95+
return spo.indexOf(key) >= 0
14596
}
14697

14798
function keyIsNotAObject (tripleKey) {
@@ -229,14 +180,14 @@ function materializer (pattern, data) {
229180
}
230181

231182
module.exports = {
232-
defs,
233-
defKeys,
183+
escapeKeyValue,
184+
unescapeKeyValue,
185+
encodeTriple,
234186
encodeKey,
235187
decodeKey,
188+
typesFromPattern,
236189
filterTriple,
237190
collect,
238-
generateBatch,
239-
createQuery,
240191
extraDataMask,
241192
queryMask,
242193
variablesMask,

0 commit comments

Comments
 (0)