Skip to content

Commit f6696dc

Browse files
cubapCopilotthehabes
authored
Separated db-controller.js into modules (#201)
* Separated db-controller.js into modules * Delete overwrite-optimistic-locking.test.js * Update update.js where credit is due * Update controllers/delete.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update controllers/utils.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Create db-controller-clean.js * Remove unused db-controller backup and clean scripts * changes from testing and reviewing --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Bryan Haberberger <bryan.j.haberberger@slu.edu>
1 parent bc06b41 commit f6696dc

File tree

15 files changed

+4870
-2371
lines changed

15 files changed

+4870
-2371
lines changed

‎controllers/bulk.js‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Bulk operations controller for RERUM operations
5+
* Handles bulk create and bulk update operations
6+
* @author Claude Sonnet 4, cubap, thehabes
7+
*/
8+
9+
import { newID, isValidID, db } from '../database/index.js'
10+
import utils from '../utils.js'
11+
import { _contextid, ObjectID, createExpressError, getAgentClaim, parseDocumentID, idNegotiation } from './utils.js'
12+
13+
/**
14+
* Create many objects at once with the power of MongoDB bulkWrite() operations.
15+
*
16+
* @see https://www.mongodb.com/docs/manual/reference/method/db.collection.bulkWrite/
17+
*/
18+
const bulkCreate = async function (req, res, next) {
19+
res.set("Content-Type", "application/json; charset=utf-8")
20+
const documents = req.body
21+
let err = {}
22+
if (!Array.isArray(documents)) {
23+
err.message = "The request body must be an array of objects."
24+
err.status = 400
25+
next(createExpressError(err))
26+
return
27+
}
28+
if (documents.length === 0) {
29+
err.message = "No action on an empty array."
30+
err.status = 400
31+
next(createExpressError(err))
32+
return
33+
}
34+
const gatekeep = documents.filter(d=> {
35+
// Each item must be valid JSON, but can't be an array.
36+
if(Array.isArray(d) || typeof d !== "object") return d
37+
try {
38+
JSON.parse(JSON.stringify(d))
39+
} catch (err) {
40+
return d
41+
}
42+
// Items must not have an @id, and in some cases same for id.
43+
const idcheck = _contextid(d["@context"]) ? (d.id ?? d["@id"]) : d["@id"]
44+
if(idcheck) return d
45+
})
46+
if (gatekeep.length > 0) {
47+
err.message = "All objects in the body of a `/bulkCreate` must be JSON and must not contain a declared identifier property."
48+
err.status = 400
49+
next(createExpressError(err))
50+
return
51+
}
52+
53+
// TODO: bulkWrite SLUGS? Maybe assign an id to each document and then use that to create the slug?
54+
// let slug = req.get("Slug")
55+
// if(slug){
56+
// const slugError = await exports.generateSlugId(slug)
57+
// if(slugError){
58+
// next(createExpressError(slugError))
59+
// return
60+
// }
61+
// else{
62+
// slug = slug_json.slug_id
63+
// }
64+
// }
65+
66+
// unordered bulkWrite() operations have better performance metrics.
67+
let bulkOps = []
68+
const generatorAgent = getAgentClaim(req, next)
69+
for(let d of documents) {
70+
// Do not create empty {}s
71+
if(Object.keys(d).length === 0) continue
72+
const providedID = d?._id
73+
const id = isValidID(providedID) ? providedID : ObjectID()
74+
d = utils.configureRerumOptions(generatorAgent, d)
75+
// id is also protected in this case, so it can't be set.
76+
if(_contextid(d["@context"])) delete d.id
77+
d._id = id
78+
d['@id'] = `${process.env.RERUM_ID_PREFIX}${id}`
79+
bulkOps.push({ insertOne : { "document" : d }})
80+
}
81+
try {
82+
let dbResponse = await db.bulkWrite(bulkOps, {'ordered':false})
83+
res.set("Content-Type", "application/json; charset=utf-8")
84+
res.set("Link",dbResponse.result.insertedIds.map(r => `${process.env.RERUM_ID_PREFIX}${r._id}`)) // https://www.rfc-editor.org/rfc/rfc5988
85+
res.status(201)
86+
const estimatedResults = bulkOps.map(f=>{
87+
let doc = f.insertOne.document
88+
doc = idNegotiation(doc)
89+
return doc
90+
})
91+
res.json(estimatedResults) // https://www.rfc-editor.org/rfc/rfc7231#section-6.3.2
92+
}
93+
catch (error) {
94+
//MongoServerError from the client has the following properties: index, code, keyPattern, keyValue
95+
next(createExpressError(error))
96+
}
97+
}
98+
99+
/**
100+
* Update many objects at once with the power of MongoDB bulkWrite() operations.
101+
* Make sure to alter object __rerum.history as appropriate.
102+
* The same object may be updated more than once, which will create history branches (not straight sticks)
103+
*
104+
* @see https://www.mongodb.com/docs/manual/reference/method/db.collection.bulkWrite/
105+
*/
106+
const bulkUpdate = async function (req, res, next) {
107+
res.set("Content-Type", "application/json; charset=utf-8")
108+
const documents = req.body
109+
let err = {}
110+
let encountered = []
111+
if (!Array.isArray(documents)) {
112+
err.message = "The request body must be an array of objects."
113+
err.status = 400
114+
next(createExpressError(err))
115+
return
116+
}
117+
if (documents.length === 0) {
118+
err.message = "No action on an empty array."
119+
err.status = 400
120+
next(createExpressError(err))
121+
return
122+
}
123+
const gatekeep = documents.filter(d => {
124+
// Each item must be valid JSON, but can't be an array.
125+
if(Array.isArray(d) || typeof d !== "object") return d
126+
try {
127+
JSON.parse(JSON.stringify(d))
128+
} catch (err) {
129+
return d
130+
}
131+
// Items must have an @id, or in some cases an id will do
132+
const idcheck = _contextid(d["@context"]) ? (d.id ?? d["@id"]) : d["@id"]
133+
if(!idcheck) return d
134+
})
135+
// The empty {}s will cause this error
136+
if (gatekeep.length > 0) {
137+
err.message = "All objects in the body of a `/bulkUpdate` must be JSON and must contain a declared identifier property."
138+
err.status = 400
139+
next(createExpressError(err))
140+
return
141+
}
142+
// unordered bulkWrite() operations have better performance metrics.
143+
let bulkOps = []
144+
const generatorAgent = getAgentClaim(req, next)
145+
for(const objectReceived of documents){
146+
// We know it has an id
147+
const idReceived = objectReceived["@id"] ?? objectReceived.id
148+
// Update the same thing twice? can vs should.
149+
// if(encountered.includes(idReceived)) continue
150+
encountered.push(idReceived)
151+
if(!idReceived.includes(process.env.RERUM_ID_PREFIX)) continue
152+
let id = parseDocumentID(idReceived)
153+
let originalObject
154+
try {
155+
originalObject = await db.findOne({"$or":[{"_id": id}, {"__rerum.slug": id}]})
156+
} catch (error) {
157+
next(createExpressError(error))
158+
return
159+
}
160+
if (null === originalObject) continue
161+
if (utils.isDeleted(originalObject)) continue
162+
id = ObjectID()
163+
let context = objectReceived["@context"] ? { "@context": objectReceived["@context"] } : {}
164+
let rerumProp = { "__rerum": utils.configureRerumOptions(generatorAgent, originalObject, true, false)["__rerum"] }
165+
delete objectReceived["__rerum"]
166+
delete objectReceived["_id"]
167+
delete objectReceived["@id"]
168+
// id is also protected in this case, so it can't be set.
169+
if(_contextid(objectReceived["@context"])) delete objectReceived.id
170+
delete objectReceived["@context"]
171+
let newObject = Object.assign(context, { "@id": process.env.RERUM_ID_PREFIX + id }, objectReceived, rerumProp, { "_id": id })
172+
bulkOps.push({ insertOne : { "document" : newObject }})
173+
if(originalObject.__rerum.history.next.indexOf(newObject["@id"]) === -1){
174+
originalObject.__rerum.history.next.push(newObject["@id"])
175+
const replaceOp = { replaceOne :
176+
{
177+
"filter" : { "_id": originalObject["_id"] },
178+
"replacement" : originalObject,
179+
"upsert" : false
180+
}
181+
}
182+
bulkOps.push(replaceOp)
183+
}
184+
}
185+
try {
186+
let dbResponse = await db.bulkWrite(bulkOps, {'ordered':false})
187+
res.set("Content-Type", "application/json; charset=utf-8")
188+
res.set("Link", dbResponse.result.insertedIds.map(r => `${process.env.RERUM_ID_PREFIX}${r._id}`)) // https://www.rfc-editor.org/rfc/rfc5988
189+
res.status(200)
190+
const estimatedResults = bulkOps.filter(f=>f.insertOne).map(f=>{
191+
let doc = f.insertOne.document
192+
doc = idNegotiation(doc)
193+
return doc
194+
})
195+
res.json(estimatedResults) // https://www.rfc-editor.org/rfc/rfc7231#section-6.3.2
196+
}
197+
catch (error) {
198+
//MongoServerError from the client has the following properties: index, code, keyPattern, keyValue
199+
next(createExpressError(error))
200+
}
201+
}
202+
203+
export { bulkCreate, bulkUpdate }

‎controllers/crud.js‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Basic CRUD operations for RERUM v1
5+
* @author Claude Sonnet 4, cubap, thehabes
6+
*/
7+
import { newID, isValidID, db } from '../database/index.js'
8+
import utils from '../utils.js'
9+
import { _contextid, idNegotiation, generateSlugId, ObjectID, createExpressError, getAgentClaim, parseDocumentID } from './utils.js'
10+
11+
/**
12+
* Create a new Linked Open Data object in RERUM v1.
13+
* Order the properties to preference @context and @id. Put __rerum and _id last.
14+
* Respond RESTfully
15+
* */
16+
const create = async function (req, res, next) {
17+
res.set("Content-Type", "application/json; charset=utf-8")
18+
let slug = ""
19+
if(req.get("Slug")){
20+
let slug_json = await generateSlugId(req.get("Slug"), next)
21+
if(slug_json.code){
22+
next(createExpressError(slug_json))
23+
return
24+
}
25+
else{
26+
slug = slug_json.slug_id
27+
}
28+
}
29+
30+
let generatorAgent = getAgentClaim(req, next)
31+
let context = req.body["@context"] ? { "@context": req.body["@context"] } : {}
32+
let provided = JSON.parse(JSON.stringify(req.body))
33+
let rerumProp = { "__rerum": utils.configureRerumOptions(generatorAgent, provided, false, false)["__rerum"] }
34+
rerumProp.__rerum.slug = slug
35+
const providedID = provided._id
36+
const id = isValidID(providedID) ? providedID : ObjectID()
37+
delete provided["__rerum"]
38+
delete provided["@id"]
39+
// id is also protected in this case, so it can't be set.
40+
if(_contextid(provided["@context"])) delete provided.id
41+
delete provided["@context"]
42+
43+
let newObject = Object.assign(context, { "@id": process.env.RERUM_ID_PREFIX + id }, provided, rerumProp, { "_id": id })
44+
console.log("CREATE")
45+
try {
46+
let result = await db.insertOne(newObject)
47+
res.set(utils.configureWebAnnoHeadersFor(newObject))
48+
newObject = idNegotiation(newObject)
49+
newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
50+
res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
51+
res.status(201)
52+
res.json(newObject)
53+
}
54+
catch (error) {
55+
//MongoServerError from the client has the following properties: index, code, keyPattern, keyValue
56+
next(createExpressError(error))
57+
}
58+
}
59+
60+
/**
61+
* Query the MongoDB for objects containing the key:value pairs provided in the JSON Object in the request body.
62+
* This will support wildcards and mongo params like {"key":{$exists:true}}
63+
* The return is always an array, even if 0 or 1 objects in the return.
64+
* */
65+
const query = async function (req, res, next) {
66+
res.set("Content-Type", "application/json; charset=utf-8")
67+
let props = req.body
68+
const limit = parseInt(req.query.limit ?? 100)
69+
const skip = parseInt(req.query.skip ?? 0)
70+
if (Object.keys(props).length === 0) {
71+
//Hey now, don't ask for everything...this can happen by accident. Don't allow it.
72+
let err = {
73+
message: "Detected empty JSON object. You must provide at least one property in the /query request body JSON.",
74+
status: 400
75+
}
76+
next(createExpressError(err))
77+
return
78+
}
79+
try {
80+
let matches = await db.find(props).limit(limit).skip(skip).toArray()
81+
matches = matches.map(o => idNegotiation(o))
82+
res.set(utils.configureLDHeadersFor(matches))
83+
res.json(matches)
84+
} catch (error) {
85+
next(createExpressError(error))
86+
}
87+
}
88+
89+
/**
90+
* Query the MongoDB for objects with the _id provided in the request body or request URL
91+
* Note this specifically checks for _id, the @id pattern is irrelevant.
92+
* Note /v1/id/{blank} does not route here. It routes to the generic 404
93+
* */
94+
const id = async function (req, res, next) {
95+
res.set("Content-Type", "application/json; charset=utf-8")
96+
let id = req.params["_id"]
97+
try {
98+
let match = await db.findOne({"$or": [{"_id": id}, {"__rerum.slug": id}]})
99+
if (match) {
100+
res.set(utils.configureWebAnnoHeadersFor(match))
101+
//Support built in browser caching
102+
res.set("Cache-Control", "max-age=86400, must-revalidate")
103+
//Support requests with 'If-Modified_Since' headers
104+
res.set(utils.configureLastModifiedHeader(match))
105+
// Include current version for optimistic locking
106+
const currentVersion = match.__rerum?.isOverwritten ?? ""
107+
res.set('Current-Overwritten-Version', currentVersion)
108+
match = idNegotiation(match)
109+
res.location(_contextid(match["@context"]) ? match.id : match["@id"])
110+
res.json(match)
111+
return
112+
}
113+
let err = {
114+
"message": `No RERUM object with id '${id}'`,
115+
"status": 404
116+
}
117+
next(createExpressError(err))
118+
} catch (error) {
119+
next(createExpressError(error))
120+
}
121+
}
122+
123+
export {
124+
create,
125+
query,
126+
id
127+
}

0 commit comments

Comments
 (0)