251 memory reins

controllers/crud.js

@@ -6,7 +6,7 @@
  */
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, idNegotiation, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
+import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
 
 /**
  * Create a new Linked Open Data object in RERUM v1.
@@ -37,7 +37,7 @@ const create = async function (req, res, next) {
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
     let context = req.body["@context"] ? { "@context": req.body["@context"] } : {}
-    let provided = JSON.parse(JSON.stringify(req.body))
+    let provided = utils.cloneObject(req.body)
     let rerumProp = { "__rerum": utils.configureRerumOptions(generatorAgent, provided, false, false)["__rerum"] }
     if(slug){
         rerumProp.__rerum.slug = slug
@@ -55,7 +55,7 @@ const create = async function (req, res, next) {
         let result = await db.insertOne(newObject)
         res.set(utils.configureWebAnnoHeadersFor(newObject))
         newObject = idNegotiation(newObject)
-        newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+        newObject.new_obj_state = utils.cloneObject(newObject)
         res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
         res.status(201)
         res.json(newObject)
@@ -74,8 +74,7 @@ const create = async function (req, res, next) {
 const query = async function (req, res, next) {
     res.set("Content-Type", "application/json; charset=utf-8")
     let props = req.body
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     if (!props || Object.keys(props).length === 0) {
         //Hey now, don't ask for everything...this can happen by accident.  Don't allow it.
         let err = {

controllers/gog.js

@@ -8,7 +8,7 @@
 
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation } from './utils.js'
+import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js'
 
 /**
  * THIS IS SPECIFICALLY FOR 'Gallery of Glosses'
@@ -27,8 +27,7 @@ const _gog_fragments_from_manuscript = async function (req, res, next) {
     if (!agent) return
     const agentID = agent.split("/").pop()
     const manID = req.body["ManuscriptWitness"]
-    const limit = parseInt(req.query.limit ?? 50)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 50)
     let err = { message: `` }
     // This request can only be made my Gallery of Glosses production apps.
     if (agentID !== "61043ad4ffce846a83e700dd") {
@@ -159,8 +158,7 @@ const _gog_glosses_from_manuscript = async function (req, res, next) {
     if (!agent) return
     const agentID = agent.split("/").pop()
     const manID = req.body["ManuscriptWitness"]
-    const limit = parseInt(req.query.limit ?? 50)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 50)
     let err = { message: `` }
     // This request can only be made my Gallery of Glosses production apps.
     if (agentID !== "61043ad4ffce846a83e700dd") {

controllers/history.js

@@ -8,7 +8,7 @@
 
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js'
+import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js'
 
 /**
  * Public facing servlet to gather for all versions downstream from a provided `key object`.
@@ -112,8 +112,9 @@ const idHeadRequest = async function (req, res, next) {
 const queryHeadRequest = async function (req, res, next) {
     res.set("Content-Type", "application/json; charset=utf-8")
     let props = req.body
+    const { limit, skip } = getPagination(req.query, 100)
     try {
-        let matches = await db.find(props).toArray()
+        let matches = await db.find(props).limit(limit).skip(skip).toArray()
         if (matches.length) {
             const size = Buffer.byteLength(JSON.stringify(matches))
             res.set("Content-Length", size)

controllers/patchSet.js

@@ -21,7 +21,7 @@ import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, al
 const patchSet = async function (req, res, next) {
     let err = { message: `` }
     res.set("Content-Type", "application/json; charset=utf-8")
-    let objectReceived = JSON.parse(JSON.stringify(req.body))
+    let objectReceived = utils.cloneObject(req.body)
     let originalContext
     let patchedObject = {}
     let generatorAgent = getAgentClaim(req, next)
@@ -50,7 +50,7 @@ const patchSet = async function (req, res, next) {
             })
         }
         else {
-            patchedObject = JSON.parse(JSON.stringify(originalObject))
+            patchedObject = utils.cloneObject(originalObject)
             if(_contextid(originalObject["@context"])) {
                 // If the original object has a context that needs id protected, make sure you don't set it.
                 delete objectReceived.id
@@ -73,7 +73,7 @@ const patchSet = async function (req, res, next) {
                 //Just hand back the object.  The resulting of setting nothing is the object from the request body.
                 res.set(utils.configureWebAnnoHeadersFor(originalObject))
                 originalObject = idNegotiation(originalObject)
-                originalObject.new_obj_state = JSON.parse(JSON.stringify(originalObject))
+                originalObject.new_obj_state = utils.cloneObject(originalObject)
                 res.location(originalObject[_contextid(originalObject["@context"]) ? "id":"@id"])
                 res.status(200)
                 res.json(originalObject)
@@ -93,7 +93,7 @@ const patchSet = async function (req, res, next) {
                     //Success, the original object has been updated.
                     res.set(utils.configureWebAnnoHeadersFor(newObject))
                     newObject = idNegotiation(newObject)
-                    newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+                    newObject.new_obj_state = utils.cloneObject(newObject)
                     res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
                     res.status(200)
                     res.json(newObject)

controllers/patchUnset.js

@@ -21,7 +21,7 @@ import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, al
 const patchUnset = async function (req, res, next) {
     let err = { message: `` }
     res.set("Content-Type", "application/json; charset=utf-8")
-    let objectReceived = JSON.parse(JSON.stringify(req.body))
+    let objectReceived = utils.cloneObject(req.body)
     let patchedObject = {}
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
@@ -49,7 +49,7 @@ const patchUnset = async function (req, res, next) {
             })
         }
         else {
-            patchedObject = JSON.parse(JSON.stringify(originalObject))
+            patchedObject = utils.cloneObject(originalObject)
             delete objectReceived._id //can't unset this
             delete objectReceived.__rerum //can't unset this
             delete objectReceived["@id"] //can't unset this
@@ -75,7 +75,7 @@ const patchUnset = async function (req, res, next) {
                 //Just hand back the object.  The resulting of unsetting nothing is the object.
                 res.set(utils.configureWebAnnoHeadersFor(originalObject))
                 originalObject = idNegotiation(originalObject)
-                originalObject.new_obj_state = JSON.parse(JSON.stringify(originalObject))
+                originalObject.new_obj_state = utils.cloneObject(originalObject)
                 res.location(originalObject[_contextid(originalObject["@context"]) ? "id":"@id"])
                 res.status(200)
                 res.json(originalObject)
@@ -97,7 +97,7 @@ const patchUnset = async function (req, res, next) {
                     //Success, the original object has been updated.
                     res.set(utils.configureWebAnnoHeadersFor(newObject))
                     newObject = idNegotiation(newObject)
-                    newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+                    newObject.new_obj_state = utils.cloneObject(newObject)
                     res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
                     res.status(200)
                     res.json(newObject)

controllers/patchUpdate.js

@@ -20,7 +20,7 @@ import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, al
 const patchUpdate = async function (req, res, next) {
     let err = { message: `` }
     res.set("Content-Type", "application/json; charset=utf-8")
-    let objectReceived = JSON.parse(JSON.stringify(req.body))
+    let objectReceived = utils.cloneObject(req.body)
     let patchedObject = {}
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
@@ -48,7 +48,7 @@ const patchUpdate = async function (req, res, next) {
             })
         }
         else {
-            patchedObject = JSON.parse(JSON.stringify(originalObject))
+            patchedObject = utils.cloneObject(originalObject)
             delete objectReceived.__rerum //can't patch this
             delete objectReceived._id //can't patch this
             delete objectReceived["@id"] //can't patch this
@@ -74,7 +74,7 @@ const patchUpdate = async function (req, res, next) {
                 //Just hand back the object.  The resulting of patching nothing is the object unchanged.
                 res.set(utils.configureWebAnnoHeadersFor(originalObject))
                 originalObject = idNegotiation(originalObject)
-                originalObject.new_obj_state = JSON.parse(JSON.stringify(originalObject))
+                originalObject.new_obj_state = utils.cloneObject(originalObject)
                 res.location(originalObject[_contextid(originalObject["@context"]) ? "id":"@id"])
                 res.status(200)
                 res.json(originalObject)
@@ -96,7 +96,7 @@ const patchUpdate = async function (req, res, next) {
                     //Success, the original object has been updated.
                     res.set(utils.configureWebAnnoHeadersFor(newObject))
                     newObject = idNegotiation(newObject)
-                    newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+                    newObject.new_obj_state = utils.cloneObject(newObject)
                     res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
                     res.status(200)
                     res.json(newObject)

controllers/putUpdate.js

@@ -22,7 +22,7 @@ import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, al
 const putUpdate = async function (req, res, next) {
     let err = { message: `` }
     res.set("Content-Type", "application/json; charset=utf-8")
-    let objectReceived = JSON.parse(JSON.stringify(req.body))
+    let objectReceived = utils.cloneObject(req.body)
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
     const idReceived = objectReceived["@id"] ?? objectReceived.id
@@ -69,7 +69,7 @@ const putUpdate = async function (req, res, next) {
                     //Success, the original object has been updated.
                     res.set(utils.configureWebAnnoHeadersFor(newObject))
                     newObject = idNegotiation(newObject)
-                    newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+                    newObject.new_obj_state = utils.cloneObject(newObject)
                     res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
                     res.status(200)
                     res.json(newObject)
@@ -107,7 +107,7 @@ const putUpdate = async function (req, res, next) {
 async function _import(req, res, next) {
     let err = { message: `` }
     res.set("Content-Type", "application/json; charset=utf-8")
-    let objectReceived = JSON.parse(JSON.stringify(req.body))
+    let objectReceived = utils.cloneObject(req.body)
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
     const id = ObjectID()
@@ -125,7 +125,7 @@ async function _import(req, res, next) {
         let result = await db.insertOne(newObject)
         res.set(utils.configureWebAnnoHeadersFor(newObject))
         newObject = idNegotiation(newObject)
-        newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+        newObject.new_obj_state = utils.cloneObject(newObject)
         res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
         res.status(200)
         res.json(newObject)

controllers/search.js

@@ -6,7 +6,7 @@
  */
 import { db } from '../database/index.js'
 import utils from '../utils.js'
-import { idNegotiation } from './utils.js'
+import { idNegotiation, getPagination } from './utils.js'
 
 /**
  * Merges and deduplicates results from multiple MongoDB Atlas Search index queries.
@@ -271,8 +271,7 @@ const searchAsWords = async function (req, res, next) {
         }
         return next(utils.createExpressError(err))
     }
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "text", options: searchOptions }, limit, skip)
     try {
         const [resultsPresi3, resultsPresi2] = await Promise.all([
@@ -358,8 +357,7 @@ const searchAsPhrase = async function (req, res, next) {
         }
         return next(utils.createExpressError(err))
     }
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "phrase", options: phraseOptions }, limit, skip)
     try {
         const [resultsPresi3, resultsPresi2] = await Promise.all([
@@ -437,8 +435,7 @@ const searchFuzzily = async function (req, res, next) {
         }
         return next(utils.createExpressError(err))
     }
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "text", options: fuzzyOptions }, limit, skip)
     try {
         const [resultsPresi3, resultsPresi2] = await Promise.all([
@@ -532,8 +529,7 @@ const searchWildly = async function (req, res, next) {
         }
         return next(utils.createExpressError(err))
     }
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     const [queryPresi3, queryPresi2] = buildDualIndexQueries(searchText, { type: "wildcard", options: wildcardOptions }, limit, skip)
     try {
         const [resultsPresi3, resultsPresi2] = await Promise.all([
@@ -626,8 +622,7 @@ const searchAlikes = async function (req, res, next) {
         }
         return next(utils.createExpressError(err))
     }
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     // Build moreLikeThis queries for both IIIF 3.0 and IIIF 2.1 indexes
     const searchQuery_presi3 = [
         {

controllers/utils.js

@@ -9,6 +9,24 @@ import utils from '../utils.js'
 
 const ObjectID = newID
 
+const MAX_QUERY_LIMIT = Number.parseInt(process.env.RERUM_MAX_QUERY_LIMIT ?? 500, 10)
+const MAX_QUERY_SKIP = Number.parseInt(process.env.RERUM_MAX_QUERY_SKIP ?? 100000, 10)
+
+function clampNonNegativeInt(value, fallback, max) {
+    const parsed = Number.parseInt(value, 10)
+    if (!Number.isFinite(parsed) || parsed < 0) return fallback
+    return parsed > max ? max : parsed
+}
+
+function getPagination(query = {}, defaultLimit = 100) {
+    const limitMax = Number.isFinite(MAX_QUERY_LIMIT) && MAX_QUERY_LIMIT > 0 ? MAX_QUERY_LIMIT : 500
+    const skipMax = Number.isFinite(MAX_QUERY_SKIP) && MAX_QUERY_SKIP >= 0 ? MAX_QUERY_SKIP : 100000
+    const safeDefaultLimit = defaultLimit > 0 ? defaultLimit : 100
+    const limit = clampNonNegativeInt(query.limit, safeDefaultLimit, limitMax)
+    const skip = clampNonNegativeInt(query.skip, 0, skipMax)
+    return { limit, skip }
+}
+
 /**
  * Check if a @context value contains a known @id-id mapping context
  *
@@ -52,7 +70,7 @@ const idNegotiation = function (resBody) {
     const _id = resBody._id
     delete resBody._id
     if(!resBody["@context"]) return resBody
-    let modifiedResBody = JSON.parse(JSON.stringify(resBody))
+    let modifiedResBody = utils.cloneObject(resBody)
     const context = { "@context": resBody["@context"] }
     if(_contextid(resBody["@context"])) {
         delete resBody["@id"]
@@ -182,7 +200,7 @@ async function getAllVersions(obj) {
     let rootObj
     if (primeID === "root") {
         //The obj passed in is root.  So it is the rootObj we need.
-        rootObj = JSON.parse(JSON.stringify(obj))
+        rootObj = utils.cloneObject(obj)
     } else if (primeID) {
         //The obj passed in knows the ID of root, grab it from Mongo
         //Use _id for indexed query performance instead of @id
@@ -447,6 +465,7 @@ async function healReleasesTree(releasing) {
 export {
     _contextid,
     idNegotiation,
+    getPagination,
     generateSlugId,
     index,
     ObjectID,