251 memory reins (#262)

* Add cloneObject helper and replace JSON clones

Introduce cloneObject in utils.js that uses globalThis.structuredClone when available and falls back to JSON.parse(JSON.stringify(...)). Replace direct JSON cloning in configureRerumOptions for configuredObject and received_options with cloneObject, and export cloneObject. This centralizes cloning logic and allows better deep-clone behavior when structuredClone is supported.

* Add pagination helpers and use cloneObject

Introduce MAX_QUERY_LIMIT and MAX_QUERY_SKIP and add clampNonNegativeInt and getPagination to safely parse and clamp limit/skip query params (with sensible defaults and env overrides). Replace ad-hoc deep copies (JSON.parse(JSON.stringify(...))) with utils.cloneObject in idNegotiation and getAllVersions to avoid issues with prototype loss and improve clarity. Export getPagination for reuse.

* Use utils.cloneObject and getPagination

Replace repeated JSON.parse(JSON.stringify(...)) deep-clones with utils.cloneObject across controllers (crud, patchSet, patchUnset, patchUpdate, putUpdate, gog) and add/get getPagination to standardize parsing of limit/skip in search, gog and crud. Adjust imports accordingly to centralize cloning and pagination logic, improving readability and consistency.

* Update history.js

* Use structuredClone for deep cloning

Replace numerous JSON.parse(JSON.stringify(...)) and utils.cloneObject calls with native structuredClone across controllers and utils for safer, more reliable deep copies. Remove the now-unused cloneObject helper from utils.js and update configureRerumOptions to use structuredClone. Also tweak clampNonNegativeInt to treat non-positive values (<= 0) as fallback, switch history HEAD handling to res.status(200).end(), and add safeBody usage in delete to avoid re-parsing request body.

* better head check

* Convert tests to todos, fix router & delete.js

Replace many end-to-end route tests with it.todo placeholders to temporarily stub numerous route test cases (files under routes/__tests__ and __tests__/routes_mounted.test.js). Update routes_mounted.test.js to safely access Express router stack via app._router?.stack ?? [] for compatibility. Resolve a merge/conflict in controllers/delete.js by removing logic that attempted to read an id from the request body and enforce that DELETE requires the id in the URL (returns a 400 if missing).

* Update package.json

* activating tests

* Convert route mount TODOs to real supertest checks

Replace placeholder todos with active integration tests that probe mounted routes via supertest. Removed the unused api_routes import and the routeExists helper; tests now perform actual HTTP requests against app for top-level paths (/v1, /client/register, /v1/id/:id, /v1/since/:id, /v1/history/:id) and for /v1/api/* endpoints using appropriate methods and content types, asserting endpoints are mounted (not 404) or return expected 404 for unknown IDs. This validates routing mounts across multiple HTTP verbs instead of relying on stack inspection.

* changes during review

---------

Co-authored-by: Bryan Haberberger <bryan.j.haberberger@slu.edu>

Author: cubap

__tests__/routes_mounted.test.js

@@ -6,111 +6,139 @@
  */
 
 import request from "supertest"
-import api_routes from "../routes/api-routes.js"
 import app from "../app.js"
 import fs from "fs"
 
-let app_stack = app.router.stack
-let api_stack = api_routes.stack
-
-/**
- * Check if a route exists in the Express app
- * @param {Array} stack - The router stack to search
- * @param {string} testPath - The path to test for
- * @returns {boolean} - True if the route exists
- */
-function routeExists(stack, testPath) {
-  for (const layer of stack) {
-    // Check if layer has matchers (Express 5)
-    if (layer.matchers && layer.matchers.length > 0) {
-      const matcher = layer.matchers[0]
-      const match = matcher(testPath)
-      if (match && match.path) return true
-    }
-    // Also check route.path directly if it exists
-    if (layer.route && layer.route.path) {
-      if (layer.route.path === testPath || layer.route.path.includes(testPath)) return true
-    }
-  }
-  return false
-}
-
 describe('Check to see that all expected top level route patterns exist.', () => {
 
-  it('/v1 -- mounted ', () => {
-    expect(routeExists(app_stack, '/v1')).toBe(true)
+  it('/v1 -- mounted ', async () => {
+    const response = await request(app).get('/v1')
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/client -- mounted ', () => {
-    expect(routeExists(app_stack, '/client')).toBe(true)
+  it('/client -- mounted ', async () => {
+    const response = await request(app).get('/client/register')
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/id/{_id} -- mounted', () => {
-    expect(routeExists(api_stack, '/id')).toBe(true)
+  it('/v1/id/{_id} -- mounted', async () => {
+    const response = await request(app).get('/v1/id/test-mounted-id')
+    // Mounted route with unknown id should 404 (not an unmapped endpoint 404)
+    expect(response.statusCode).toBe(404)
   })
 
-  it('/v1/since/{_id} -- mounted', () => {
-    expect(routeExists(api_stack, '/since')).toBe(true)
+  it('/v1/since/{_id} -- mounted', async () => {
+    const response = await request(app).get('/v1/since/test-mounted-id')
+    // Mounted route with unknown id should 404
+    expect(response.statusCode).toBe(404)
   })
 
-  it('/v1/history/{_id} -- mounted', () => {
-    expect(routeExists(api_stack, '/history')).toBe(true)
+  it('/v1/history/{_id} -- mounted', async () => {
+    const response = await request(app).get('/v1/history/test-mounted-id')
+    // Mounted route with unknown id should 404
+    expect(response.statusCode).toBe(404)
   })
 
 })
 
 describe('Check to see that all /v1/api/ route patterns exist.', () => {
 
-  it('/v1/api/query -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/query')).toBe(true)
+  it('/v1/api/query -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/query')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/create -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/create')).toBe(true)
+  it('/v1/api/create -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/create')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/bulkCreate -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/bulkCreate')).toBe(true)
+  it('/v1/api/bulkCreate -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/bulkCreate')
+      .set('Content-Type', 'application/json')
+      .send([{ mounted: true }])
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/update -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/update')).toBe(true)
+  it('/v1/api/update -- mounted ', async () => {
+    const response = await request(app)
+      .put('/v1/api/update')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/bulkUpdate -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/bulkUpdate')).toBe(true)
+  it('/v1/api/bulkUpdate -- mounted ', async () => {
+    const response = await request(app)
+      .put('/v1/api/bulkUpdate')
+      .set('Content-Type', 'application/json')
+      .send([{ mounted: true }])
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/overwrite -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/overwrite')).toBe(true)
+  it('/v1/api/overwrite -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/overwrite')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/patch -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/patch')).toBe(true)
+  it('/v1/api/patch -- mounted ', async () => {
+    const response = await request(app)
+      .patch('/v1/api/patch')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/set -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/set')).toBe(true)
+  it('/v1/api/set -- mounted ', async () => {
+    const response = await request(app)
+      .patch('/v1/api/set')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/unset -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/unset')).toBe(true)
+  it('/v1/api/unset -- mounted ', async () => {
+    const response = await request(app)
+      .patch('/v1/api/unset')
+      .set('Content-Type', 'application/json')
+      .send({ mounted: true })
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/delete/{id} -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/delete')).toBe(true)
+  it('/v1/api/delete/{id} -- mounted ', async () => {
+    const response = await request(app).delete('/v1/api/delete/test-mounted-id')
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/release/{id} -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/release')).toBe(true)
+  it('/v1/api/release/{id} -- mounted ', async () => {
+    const response = await request(app).patch('/v1/api/release/test-mounted-id')
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/search -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/search')).toBe(true)
+  it('/v1/api/search -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/search')
+      .set('Content-Type', 'text/plain')
+      .send('mounted search')
+    expect(response.statusCode).not.toBe(404)
   })
 
-  it('/v1/api/search/phrase -- mounted ', () => {
-    expect(routeExists(api_stack, '/api/search/phrase')).toBe(true)
+  it('/v1/api/search/phrase -- mounted ', async () => {
+    const response = await request(app)
+      .post('/v1/api/search/phrase')
+      .set('Content-Type', 'text/plain')
+      .send('mounted phrase search')
+    expect(response.statusCode).not.toBe(404)
   })
 
 })
@@ -142,4 +170,4 @@ describe('Check to see that critical repo files are present', () => {
     expect(fs.existsSync(filePath+"jest.config.js")).toBeTruthy()
     expect(fs.existsSync(filePath+"package.json")).toBeTruthy()
   })
-})
+})

controllers/bulk.js

@@ -33,7 +33,8 @@ const bulkCreate = async function (req, res, next) {
         // Each item must be valid JSON, but can't be an array.
         if(Array.isArray(d) || typeof d !== "object") return d
         try {
-            JSON.parse(JSON.stringify(d))
+            // Validate that the entry is structured-cloneable; result is intentionally discarded.
+            structuredClone(d)
         } catch (err) {
             return d
         }
@@ -120,7 +121,8 @@ const bulkUpdate = async function (req, res, next) {
         // Each item must be valid JSON, but can't be an array.
         if(Array.isArray(d) || typeof d !== "object") return d
         try {
-            JSON.parse(JSON.stringify(d))
+            // Validate that the entry is structured-cloneable; result is intentionally discarded.
+            structuredClone(d)
         } catch (err) {
             return d
         }

controllers/crud.js

@@ -6,7 +6,7 @@
  */
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, idNegotiation, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
+import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
 
 /**
  * Create a new Linked Open Data object in RERUM v1.
@@ -37,7 +37,7 @@ const create = async function (req, res, next) {
     let generatorAgent = getAgentClaim(req, next)
     if (!generatorAgent) return
     let context = req.body["@context"] ? { "@context": req.body["@context"] } : {}
-    let provided = JSON.parse(JSON.stringify(req.body))
+    let provided = structuredClone(req.body)
     let rerumProp = { "__rerum": utils.configureRerumOptions(generatorAgent, provided, false, false)["__rerum"] }
     if(slug){
         rerumProp.__rerum.slug = slug
@@ -55,7 +55,7 @@ const create = async function (req, res, next) {
         let result = await db.insertOne(newObject)
         res.set(utils.configureWebAnnoHeadersFor(newObject))
         newObject = idNegotiation(newObject)
-        newObject.new_obj_state = JSON.parse(JSON.stringify(newObject))
+        newObject.new_obj_state = structuredClone(newObject)
         res.location(newObject[_contextid(newObject["@context"]) ? "id":"@id"])
         res.status(201)
         res.json(newObject)
@@ -74,8 +74,7 @@ const create = async function (req, res, next) {
 const query = async function (req, res, next) {
     res.set("Content-Type", "application/json; charset=utf-8")
     let props = req.body
-    const limit = parseInt(req.query.limit ?? 100)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 100)
     if (!props || Object.keys(props).length === 0) {
         //Hey now, don't ask for everything...this can happen by accident.  Don't allow it.
         let err = {

controllers/delete.js

@@ -33,7 +33,7 @@ const deleteObj = async function(req, res, next) {
         return next(utils.createExpressError(error))
     }
     if (null !== originalObject) {
-        let safe_original = JSON.parse(JSON.stringify(originalObject))
+        let safe_original = structuredClone(originalObject)
         if (utils.isDeleted(safe_original)) {
             err = Object.assign(err, {
                 message: `The object you are trying to delete is already deleted. ${err.message}`,
@@ -57,7 +57,7 @@ const deleteObj = async function(req, res, next) {
         }
         let preserveID = safe_original["@id"]
         let deletedFlag = {} //The __deleted flag is a JSONObject
-        deletedFlag["object"] = JSON.parse(JSON.stringify(originalObject))
+        deletedFlag["object"] = structuredClone(originalObject)
         deletedFlag["deletor"] = agentRequestingDelete
         deletedFlag["time"] = new Date(Date.now()).toISOString().replace("Z", "")
         let deletedObject = {
@@ -121,7 +121,7 @@ async function healHistoryTree(obj) {
             const nextIdForQuery = parseDocumentID(nextID)
             const objToUpdate = await db.findOne({"$or":[{"_id": nextIdForQuery}, {"__rerum.slug": nextIdForQuery}]})
             if (null !== objToUpdate) {
-                let fixHistory = JSON.parse(JSON.stringify(objToUpdate))
+                let fixHistory = structuredClone(objToUpdate)
                 if (objToDeleteisRoot) {
                     //This means this next object must become root. 
                     //Strictly, all history trees must have num(root) > 0.  
@@ -154,7 +154,7 @@ async function healHistoryTree(obj) {
             let previousIdForQuery = parseDocumentID(previous_id)
             const objToUpdate2 = await db.findOne({"$or":[{"_id": previousIdForQuery}, {"__rerum.slug": previousIdForQuery}]})
             if (null !== objToUpdate2) {
-                let fixHistory2 = JSON.parse(JSON.stringify(objToUpdate2))
+                let fixHistory2 = structuredClone(objToUpdate2)
                 let origNextArray = fixHistory2["__rerum"]["history"]["next"]
                 let newNextArray = [...origNextArray]
                 newNextArray = newNextArray.filter(id => id !== obj["@id"])
@@ -193,7 +193,7 @@ async function newTreePrime(obj) {
             // fail silently
         }
         for (const d of descendants) {
-            let objWithUpdate = JSON.parse(JSON.stringify(d))
+            let objWithUpdate = structuredClone(d)
             objWithUpdate["__rerum"]["history"]["prime"] = primeID
             let result = await db.replaceOne({ "_id": d["_id"] }, objWithUpdate)
             if (result.modifiedCount === 0) {

controllers/gog.js

@@ -8,7 +8,7 @@
 
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation } from './utils.js'
+import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js'
 
 /**
  * THIS IS SPECIFICALLY FOR 'Gallery of Glosses'
@@ -27,8 +27,7 @@ const _gog_fragments_from_manuscript = async function (req, res, next) {
     if (!agent) return
     const agentID = agent.split("/").pop()
     const manID = req.body["ManuscriptWitness"]
-    const limit = parseInt(req.query.limit ?? 50)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 50)
     let err = { message: `` }
     // This request can only be made my Gallery of Glosses production apps.
     if (agentID !== "61043ad4ffce846a83e700dd") {
@@ -159,8 +158,7 @@ const _gog_glosses_from_manuscript = async function (req, res, next) {
     if (!agent) return
     const agentID = agent.split("/").pop()
     const manID = req.body["ManuscriptWitness"]
-    const limit = parseInt(req.query.limit ?? 50)
-    const skip = parseInt(req.query.skip ?? 0)
+    const { limit, skip } = getPagination(req.query, 50)
     let err = { message: `` }
     // This request can only be made my Gallery of Glosses production apps.
     if (agentID !== "61043ad4ffce846a83e700dd") {
@@ -389,7 +387,7 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde
     })
 
     // Combine the Annotation bodies with the primitive object
-    let expandedEntity = JSON.parse(JSON.stringify(primitiveEntity))
+    let expandedEntity = structuredClone(primitiveEntity)
     for(const anno of matches){
         const body = anno.body
         let keys = Object.keys(body)