@id and id negotiation (#191)

thehabes · dependabot[bot] · web-flow · commit b55cecde063b · 2025-04-15T14:30:33.000-05:00
* Good id negotiation in the db controller for a create request/response * touch em all * touch em all * touch em all * Get rid of _id regularly * Since it is somewhat likely @context will be an array * tests * Tests we will need TODO * Good error throughput * Changes from manual testing * Error redo * Error redo * Error redo * Now errors are handled in rest.js without unnecessary interceptions * documentation and cleanup * documentation and cleanup * documentation and cleanup * can put this back now * can put this back now * can put this back now * 65 bulk update (#192) * The shape of it * straighten out errors, take into other branch * ew the nasties * Register the bulkUpdate route. Make the errors behave. * Functional bulkUpdate and much improved error reporting. * bulkUpdate test entries. skipping end to end test for it, for now. * Harder checks against objects supplied in the bodies of bulk endpoints. * Harder checks against objects supplied in the bodies of bulk endpoints. * ah ')' * overachieving just a lil bit * Performance update for bulkCreate and bulkUpdate, which may not process in the same order as submitted. * Bump @babel/helpers from 7.23.5 to 7.27.0 (#193) Bumps [@babel/helpers](https://github.com/babel/babel/tree/HEAD/packages/babel-helpers) from 7.23.5 to 7.27.0. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.27.0/packages/babel-helpers) --- updated-dependencies: - dependency-name: "@babel/helpers" dependency-version: 7.27.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * no prezi 2 * yes oa * cmon * lint this bad indent * formally skip() this test * API wording around /bulkUpdate and /bulkCreate. * oh boy missed this * oh boy missed this * oh boy missed this --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
diff --git a/__tests__/routes_mounted.test.js b/__tests__/routes_mounted.test.js
@@ -120,6 +120,19 @@ describe('Check to see that all /v1/api/ route patterns exist.', () => {
     expect(exists).toBe(true)
   })
 
+  it('/v1/api/bulkUpdate -- mounted ', () => {
+   let exists = false
+    for (const middleware of api_stack) {
+      if (middleware.regexp 
+        && middleware.regexp.toString().includes("/api")
+        && middleware.regexp.toString().includes("/bulkUpdate")){
+        exists = true
+        break
+      }
+    }
+    expect(exists).toBe(true)
+  })
+
   it('/v1/api/patch -- mounted ', () => {
    let exists = false
     for (const middleware of api_stack) {
diff --git a/database/index.js b/database/index.js
@@ -4,6 +4,7 @@ dotenv.config()
 
 const client = new MongoClient(process.env.MONGO_CONNECTION_STRING)
 const newID = () => new ObjectId().toHexString()
+const isValidID = (id) => ObjectId.isValid(id)
 const connected = async function () {
     // Send a ping to confirm a successful connection
     await client.db("admin").command({ ping: 1 }).catch(err => err)
@@ -50,6 +51,7 @@ function isValidURL(url) {
 
 export {
     newID,
+    isValidID,
     connected,
     db
 }
diff --git a/db-controller.js b/db-controller.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/public/API.html b/public/API.html
@@ -420,10 +420,11 @@ <h3 id="bulk-create">Bulk Create</h3>
 
         <p>
             Add multiple completely new objects to RERUM and receive an array of the complete records as the response body. 
-            Accepts only a single array of JSON objects in the request body.
-            The array of JSON objects passed in will be created in the order submitted and the response will have the URI 
-            of the new resource or an error message as an array in the same order. When errors are encountered, 
-            the batch process will attempt to continue for all submitted items.
+            Accepts only a single array of JSON objects in the request body.  The '@id' property must not be present on the objects.
+            In cases where the Linked Data @context property maps '@id' to 'id', the 'id' property also must not be present.
+            The array of JSON objects passed may not be created in the order submitted.  The response will have the URI 
+            of the new resource or an error message as an array in the order the order the objects were processed. 
+            When errors are encountered, the batch process will attempt to continue for all submitted items.
         </p>
 
         <p> 
@@ -730,6 +731,80 @@ <h3 id="update">Update</h3>
             </p>
         </p>
 
+        <h3 id="bulk-update">Bulk Update</h3>
+
+        <table>
+            <thead>
+                <tr>
+                    <th>Patterns</th>
+                    <th>Payloads</th>
+                    <th>Responses</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td><code class="language-plaintext highlighter-rouge">/bulkUpdate</code></td>
+                    <td><code class="language-plaintext highlighter-rouge">[{JSON}]</code></td>
+                    <td>201 <code>
+                        <code class="language-plaintext highlighter-rouge">[{JSON}]</code></td>
+                </tr>
+            </tbody>
+        </table>
+
+        <ul>
+            <li><strong><code class="language-plaintext highlighter-rouge">[{JSON}]</code></strong>—an array RERUM objects
+                to be updated.</li>
+            <li><strong>Response: <code class="language-plaintext highlighter-rouge">[{JSON}]</code></strong>—an array
+                of the resolved records from the update process</li>
+        </ul>
+
+        <p>
+            Update multiple existing RERUM objects at once and recieve an array of the complete records as the response body.
+            Accepts only a single array of JSON objects in the request body.  The '@id' property must be present for each object.
+            In cases where the Linked Data @context property maps '@id' to 'id' either of these properties will be sufficient.
+            The array of JSON objects passed in may not be updated in the order submitted.  The response will have the URI 
+            of the new resource or an error message as an array in the order the objects were processed. When errors are encountered, the batch process will attempt to continue for all submitted items.
+        </p>
+
+        <p> 
+            <div class="exHeading">Javascript Example</div>
+            <pre><code class="jsExample"> 
+                <span>const saved_obj = await fetch("https://devstore.rerum.io/v1/api/bulkUpdate", {</span>
+                    <span class="ind1">method: "POST",</span>
+                    <span class="ind1">headers:{</span>
+                        <span class="ind2">"Authorization": "Bearer eyJz93a...k4laUWw" </span>
+                        <span class="ind2">"Content-Type": "application/json; charset=utf-8"</span>
+                    <span class="ind1">},</span>
+                    <span class="ind1">body: JSON.stringify([
+                        <span class="ind2">"@id": "https://devstore.rerum.io/v1/id/abcdef1234567890",</span>
+                        <span class="ind2">"hello": "new world",</span>
+                        <span class="ind2">"@id": "https://devstore.rerum.io/v1/id/1234567890abcdef",</span>
+                        <span class="ind2">"goodbye": "old planet"</span>
+                    <span class="ind1">])</span>
+                <span>})</span>
+                <span>.then(resp => resp.json())</span>
+                <span>.catch(err => {throw err})
+            </code></pre>
+        </p>
+
+        <p>
+            <div class="exHeading">Here is what the response <code>resp</code> looks like:</div>
+            <pre><code class="respExample">
+                <span>[</span>
+                    <span class="ind1">{</span>
+                        <span class="ind2">"@id": "https://devstore.rerum.io/v1/id/abcabc1231231230",</span>
+                        <span class="ind2">"hello": "new world",</span>
+                        <span class="ind2">"__rerum":{...}</span>
+                    <span class="ind1">},</span>
+                    <span class="ind1">{</span>
+                        <span class="ind2">"@id": "https://devstore.rerum.io/v1/id/defdef4564564567",</span>
+                        <span class="ind2">"goodbye": "old planet",</span>
+                        <span class="ind2"> "__rerum":{...}</span>
+                    <span class="ind1">}</span>
+                <span>]</span>
+            </code></pre>
+        </p>
+
         <h3 id="overwrite">Overwrite</h3>
 
         <p>
diff --git a/rest.js b/rest.js
@@ -29,54 +29,52 @@ const checkPatchOverrideSupport = function (req, res) {
  * REST is all about communication.  The response code and the textual body are particular.
  * RERUM is all about being clear.  It will build custom responses sometimes for certain scenarios, will remaining RESTful.
  * 
- * Note that the res upstream from this has been converted into err.  res will not have what you are looking for, check err instead. 
+ * You have likely reached this with a next(createExpressError(err)) call.  End here and send the error.
  */
 const messenger = function (err, req, res, next) {
     if (res.headersSent) {
-        next(err)
         return
     }
-    err.message = err.message ?? res.message ?? ``
-    if (err.statusCode === 401) {
+    let error = {}
+    error.message = err.statusMessage ?? err.message ?? ``
+    error.status = err.statusCode ?? err.status ?? 500
+    if (error.status === 401) {
         //Special handler for token errors from the oauth module
         //Token errors come through with a message that we want.  That message is in the error's WWW-Authenticate header
         //Other 401s from our app come through with a status message.  They may not have headers.
         if (err.headers?.["WWW-Authenticate"]) {
-            err.message += err.headers["WWW-Authenticate"]
+            error.message += err.headers["WWW-Authenticate"]
         }
     }
-    let genericMessage = ""
     let token = req.header("Authorization")
     if(token && !token.startsWith("Bearer ")){
-        err.message +=`
+        error.message +=`
 Your token is not in the correct format.  It should be a Bearer token formatted like: "Bearer <token>"`
-        next(err)
-        return
     }
-    switch (err.statusCode) {
+    switch (error.status) {
         case 400:
             //"Bad Request", most likely because the body and Content-Type are not aligned.  Could be bad JSON.
-            err.message += `
+            error.message += `
 The body of your request was invalid. Please make sure it is a valid content-type and that the body matches that type.
 If the body is JSON, make sure it is valid JSON.`
             break
         case 401:
             //The requesting agent is known from the request.  That agent does not match __rerum.generatedBy.  Unauthorized.
             if (token) {
-                err.message += `
+                error.message += `
 The token provided is Unauthorized.  Please check that it is your token and that it is not expired. 
 Token: ${token} `
             }
             else {
-                err.message += `
+                error.message += `
 The request does not contain an "Authorization" header and so is Unauthorized. Please include a token with your requests
 like "Authorization: Bearer <token>". Make sure you have registered at ${process.env.RERUM_PREFIX}.`
             }
             break
         case 403:
             //Forbidden to use this.  The provided Bearer does not have the required privileges. 
             if (token) {
-                err.message += `
+                error.message += `
 You are Forbidden from performing this action.  Check your privileges.
 Token: ${token}`
             }
@@ -87,24 +85,31 @@ You are Forbidden from performing this action. The request does not contain an "
 Make sure you have registered at ${process.env.RERUM_PREFIX}. `
             }
         case 404:
-            err.message += `
+            error.message += `
 The requested web page or resource could not be found.`
             break
         case 405:
             // These are all handled in api-routes.js already.
             break
+        case 409:
+            // These are all handled in db-controller.js already.
+            break
+        case 501:
+            // Not implemented.  Handled upstream.
+            break
         case 503:
             //RERUM is down or readonly.  Handled upstream.
             break
         case 500:
         default:
             //Really bad, probably not specifically caught.  
-            err.message += `
+            error.message += `
 RERUM experienced a server issue while performing this action.
 It may not have completed at all, and most likely did not complete successfully.`
     }
-    //    res.status(statusCode).send(err.statusMessage)
-    next(err)
+    console.error(error)
+    res.set("Content-Type", "text/plain; charset=utf-8")
+    res.status(error.status).send(error.message)
 }
 
 export default { checkPatchOverrideSupport, messenger }
diff --git a/routes/__tests__/bulkUpdate.test.js b/routes/__tests__/bulkUpdate.test.js
@@ -0,0 +1,23 @@
+import { jest } from "@jest/globals"
+
+// Only real way to test an express route is to mount it and call it so that we can use the req, res, next.
+import express from "express"
+import request from "supertest"
+import controller from '../../db-controller.js'
+
+// Here is the auth mock so we get a req.user and the controller can function without a NPE.
+const addAuth = (req, res, next) => {
+  req.user = {"http://store.rerum.io/agent": "https://store.rerum.io/v1/id/agent007"}
+  next()
+}
+
+const routeTester = new express()
+routeTester.use(express.json())
+routeTester.use(express.urlencoded({ extended: false }))
+
+// Mount our own /bulkCreate route without auth that will use controller.bulkCreate
+routeTester.use("/bulkUpdate", [addAuth, controller.bulkUpdate])
+
+it.skip("'/bulkUpdate' route functions", async () => {
+  // TODO without hitting the v1/id/11111 object because it is already abused.
+})
diff --git a/routes/__tests__/create.test.js b/routes/__tests__/create.test.js
@@ -38,3 +38,7 @@ it("'/create' route functions", async () => {
   expect(response.headers["link"]).toBeTruthy()
 
 })
+
+it.skip("Support setting valid '_id' on '/create' request body.", async () => {
+  // TODO
+})
diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
@@ -32,3 +32,7 @@ it("'/id/:id' route functions", async () => {
   expect(response.headers["location"]).toBeTruthy()
 
 })
+
+it.skip("Proper '@id-id' negotation on GET by URI.", async () => {
+  // TODO
+})
diff --git a/routes/__tests__/idNegotiation.test.js b/routes/__tests__/idNegotiation.test.js
@@ -0,0 +1,30 @@
+import { jest } from "@jest/globals"
+import dotenv from "dotenv"
+import controller from '../../db-controller.js'
+
+it("Functional '@id-id' negotiation on objects returned.", async () => {
+  let negotiate = {
+    "@context": "http://iiif.io/api/presentation/3/context.json",
+    "_id": "example",
+    "@id": `${process.env.RERUM_ID_PREFIX}example`,
+    "test": "item"
+  }
+  negotiate = controller.idNegotiation(negotiate)
+  expect(negotiate._id).toBeUndefined()
+  expect(negotiate["@id"]).toBeUndefined()
+  expect(negotiate.id).toBe(`${process.env.RERUM_ID_PREFIX}example`)
+  expect(negotiate.test).toBe("item")
+
+  let nonegotiate = {
+    "@context":"http://example.org/context.json",
+    "_id": "example",
+    "@id": `${process.env.RERUM_ID_PREFIX}example`,
+    "id": "test_example",
+    "test":"item"
+  }
+  nonegotiate = controller.idNegotiation(nonegotiate)
+  expect(nonegotiate._id).toBeUndefined()
+  expect(nonegotiate["@id"]).toBe(`${process.env.RERUM_ID_PREFIX}example`)
+  expect(nonegotiate.id).toBe("test_example")
+  expect(nonegotiate.test).toBe("item")
+})
diff --git a/routes/__tests__/query.test.js b/routes/__tests__/query.test.js
@@ -31,3 +31,7 @@ it("'/query' route functions", async () => {
     expect(response.headers["link"]).toBeTruthy()
  
 })
+
+it.skip("Proper '@id-id' negotation on objects returned from '/query'.", async () => {
+  // TODO
+})
diff --git a/routes/api-routes.js b/routes/api-routes.js
@@ -22,6 +22,8 @@ import queryRouter from './query.js';
 import createRouter from './create.js';
 // Support POST requests with JSON Array bodies used for establishing new objects.
 import bulkCreateRouter from './bulkCreate.js';
+//Support PUT requests with JSON Array bodies used for updating a number of existing objects.
+import bulkUpdateRouter from './bulkUpdate.js';
 // Support DELETE requests like v1/delete/{object id} to mark an object as __deleted.
 import deleteRouter from './delete.js';
 // Support POST requests with JSON bodies used for replacing some existing object.
@@ -47,6 +49,7 @@ router.use('/api', compatabilityRouter)
 router.use('/api/query', queryRouter)
 router.use('/api/create', createRouter)
 router.use('/api/bulkCreate', bulkCreateRouter)
+router.use('/api/bulkUpdate', bulkUpdateRouter)
 router.use('/api/delete', deleteRouter)
 router.use('/api/overwrite', overwriteRouter)
 router.use('/api/update', updateRouter)
@@ -73,22 +76,8 @@ router.get('/api', (req, res) => {
 })
 router.use('/since', sinceRouter)
 router.use('/history', historyRouter)
-/**
- * Use this to catch 404s because of invalid /api/ paths and pass them to the error handler in app.js
- * 
- * Note while we have 501s, they will fall here.  Don't let them trick you.
- * Detect them and send them out, don't hand up to the 404 catcher in app.js
- */
-router.use((req, res, next) => {
-    if (res.statusCode === 501) {
-        //We can remove this once we implement the functions, for now we have to catch it here.
-        let msg = res.statusMessage ?? "This is not yet implemented"
-        res.status(501).send(msg).end()
-    }
-    else {
-        //A 404 to pass along to our 404 handler in app.js
-        next()
-    }
-})
+
+// Note that error responses are handled by rest.js through app.js.  No need to do anything with them here.
+
 // Export API routes
 export default router
diff --git a/routes/bulkUpdate.js b/routes/bulkUpdate.js
@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+import express from 'express'
+const router = express.Router()
+
+//This controller will handle all MongoDB interactions.
+import controller from '../db-controller.js'
+import auth from '../auth/index.js'
+
+router.route('/')
+    .put(auth.checkJwt, controller.bulkUpdate)
+    .all((req, res, next) => {
+        res.statusMessage = 'Improper request method for creating, please use PUT.'
+        res.status(405)
+        next(res)
+    })
+
+export default router
diff --git a/routes/patchSet.js b/routes/patchSet.js
@@ -14,7 +14,7 @@ router.route('/')
         else {
             res.statusMessage = 'Improper request method for updating, please use PATCH to add new keys to this object.'
             res.status(405)
-            next()
+            next(res)
         }
     }) 
     .all((req, res, next) => {
diff --git a/routes/patchUnset.js b/routes/patchUnset.js
@@ -14,7 +14,7 @@ router.route('/')
         else {
             res.statusMessage = 'Improper request method for updating, please use PATCH to remove keys from this object.'
             res.status(405)
-            next()
+            next(res)
         }
     }) 
     .all((req, res, next) => {
diff --git a/routes/patchUpdate.js b/routes/patchUpdate.js
