Merge pull request #3 from exogen/request-body

Add request body support

Author: exogen

README.md

@@ -1,8 +1,8 @@
 # node-fetch-har
 
-[![npm](https://img.shields.io/npm/v/node-fetch-har.svg)](https://www.npmjs.com/package/node-fetch-har)
-![Travis](https://img.shields.io/travis/exogen/node-fetch-har.svg)
-![Coveralls](https://img.shields.io/coveralls/github/exogen/node-fetch-har.svg)
+[![npm](https://img.shields.io/npm/v/node-fetch-har.svg)][npm]
+[![Travis](https://img.shields.io/travis/exogen/node-fetch-har.svg)][travis]
+[![Coveralls](https://img.shields.io/coveralls/github/exogen/node-fetch-har.svg)][coveralls]
 
 A [Fetch API][fetch] wrapper that records [HAR logs][har] for server requests
 made with [node-fetch][]. You can then expose this data to get visibility into
@@ -129,15 +129,30 @@ if (!process.browser) {
 
 ### Redirects
 
+Due to redirects, it is possible for a single `fetch` call to result in multiple
+HTTP requests. As you might expect, multiple HAR entries will be recorded as
+well.
+
 With the Fetch API’s `redirect` option in `follow` mode (the default), calls
-will transparently follow redirects; that is, you get the response from the
+will transparently follow redirects; that is, you get the response for the
 final, redirected request. Likewise, the `harEntry` property of the response
 will correspond with that final request.
 
 To get the HAR entries for the redirects, use the `har` or `onHarEntry` options
 (described above). The redirects will be appended to the log and reported with
-`onHarEntry` along with the final entry. Note that this means that it’s possible
-for a single `fetch` call to result in multiple entries.
+`onHarEntry` in addition to the final entry, in the order that they were made.
+
+### Request Body
+
+If there is no `Content-Type` header specified in the request, then `postData`
+will not be populated since we would not be able to populate the required
+`mimeType` field.
+
+Additionally, `params` will only be populated if the `Content-Type` is exactly
+`application/x-www-form-urlencoded`. If it is anything else (including
+`multipart/form-data`) then `text` will be populated instead.
+
+There may be limited support for exotic request body encodings.
 
 ### Page Info
 
@@ -183,7 +198,6 @@ $ yarn start
 
 ## TODO
 
-- Support for request body info.
 - Support for compression info.
 - Better tests with multiple response types, encodings, etc.
 
@@ -199,3 +213,6 @@ key timestamps and metadata like the HTTP version.
 [har]: http://www.softwareishard.com/blog/har-12-spec/
 [isomorphic-fetch]: https://github.com/matthew-andrews/isomorphic-fetch
 [isomorphic-unfetch]: https://github.com/developit/unfetch
+[npm]: https://www.npmjs.com/package/node-fetch-har
+[travis]: https://travis-ci.org/exogen/node-fetch-har
+[coveralls]: https://coveralls.io/github/exogen/node-fetch-har

index.js

@@ -1,6 +1,7 @@
 const { URL } = require("url");
 const http = require("http");
 const https = require("https");
+const querystring = require("querystring");
 const generateId = require("nanoid");
 const cookie = require("cookie");
 const setCookie = require("set-cookie-parser");
@@ -32,6 +33,7 @@ function handleRequest(harEntryMap, request, options) {
   const url = new URL(options.url || options.href); // Depends on Node version?
 
   const entry = {
+    _parent: parentEntry,
     _timestamps: {
       start: now,
       sent: now
@@ -54,14 +56,14 @@ function handleRequest(harEntryMap, request, options) {
       ssl: -1
     },
     request: {
-      url: url.href,
       method: request.method,
+      url: url.href,
+      cookies: buildRequestCookies(headers),
+      headers: buildHeaders(headers),
       queryString: [...url.searchParams].map(([name, value]) => ({
         name,
         value
       })),
-      cookies: buildRequestCookies(headers),
-      headers: buildHeaders(headers),
       headersSize: -1,
       bodySize: -1
     },
@@ -71,28 +73,87 @@ function handleRequest(harEntryMap, request, options) {
     }
   };
 
-  if (parentEntry) {
-    entry._parent = parentEntry;
-  }
+  // Some versions of `node-fetch` will put `body` in the `options` received by
+  // this function and others exclude it. Instead we have to capture writes to
+  // the `ClientRequest` stream. There might be some official way to do this
+  // with streams, but the events and piping I tried didn't work. FIXME?
+  const _write = request.write;
+  const _end = request.end;
+  let requestBody;
+
+  const concatBody = chunk => {
+    // Assume the writer will be consistent such that we wouldn't get Buffers in
+    // some writes and strings in others.
+    if (typeof chunk === "string") {
+      if (requestBody == null) {
+        requestBody = chunk;
+      } else {
+        requestBody += chunk;
+      }
+    } else if (Buffer.isBuffer(chunk)) {
+      if (requestBody == null) {
+        requestBody = chunk;
+      } else {
+        requestBody = Buffer.concat([requestBody, chunk]);
+      }
+    }
+  };
+
+  request.write = function(...args) {
+    concatBody(...args);
+    return _write.call(this, ...args);
+  };
+
+  request.end = function(...args) {
+    concatBody(...args);
 
-  entry.request.url = url.href;
+    if (requestBody != null) {
+      // Works for both buffers and strings.
+      entry.request.bodySize = requestBody.length;
+
+      let mimeType;
+      for (const name in headers) {
+        if (name.toLowerCase() === "content-type") {
+          mimeType = headers[name][0];
+          break;
+        }
+      }
+
+      if (mimeType) {
+        const bodyString = requestBody.toString(); // FIXME: Assumes encoding?
+        if (mimeType === "application/x-www-form-urlencoded") {
+          entry.request.postData = {
+            mimeType,
+            params: buildParams(bodyString)
+          };
+        } else {
+          entry.request.postData = { mimeType, text: bodyString };
+        }
+      }
+    }
+
+    return _end.call(this, ...args);
+  };
 
   request.on("response", response => {
     entry._timestamps.firstByte = Date.now();
     harEntryMap.set(requestId, entry);
     const httpVersion = `HTTP/${response.httpVersion}`;
+
+    // Populate request info that isn't available until now.
     entry.request.httpVersion = httpVersion;
+
     entry.response = {
-      httpVersion,
       status: response.statusCode,
       statusText: response.statusMessage,
-      redirectURL: response.headers.location || "",
-      headers: buildHeaders(response.rawHeaders),
+      httpVersion,
       cookies: buildResponseCookies(response.headers),
+      headers: buildHeaders(response.rawHeaders),
       content: {
         size: -1,
         mimeType: response.headers["content-type"]
       },
+      redirectURL: response.headers.location || "",
       headersSize: -1,
       bodySize: -1
     };
@@ -145,6 +206,22 @@ function buildRequestCookies(headers) {
   return cookies;
 }
 
+function buildParams(paramString) {
+  const params = [];
+  const parsed = querystring.parse(paramString);
+  for (const name in parsed) {
+    const value = parsed[name];
+    if (Array.isArray(value)) {
+      value.forEach(item => {
+        params.push({ name, value: item });
+      });
+    } else {
+      params.push({ name, value });
+    }
+  }
+  return params;
+}
+
 function buildResponseCookies(headers) {
   const cookies = [];
   const setCookies = headers["set-cookie"];

test.js

@@ -380,6 +380,43 @@ fragment TypeRef on __Type {
           );
           expect(response.harEntry.request.method).toBe("POST");
         });
+
+        it("records request body info", async () => {
+          const fetch = withHar(baseFetch);
+          const response = await fetch("https://postman-echo.com/post", {
+            method: "POST",
+            headers: {
+              "Content-Type": "text/plain"
+            },
+            body: "test one two!"
+          });
+          expect(response.harEntry.request.bodySize).toBe(13);
+          expect(response.harEntry.request.postData).toEqual({
+            mimeType: "text/plain",
+            text: "test one two!"
+          });
+        });
+
+        it("records request body params", async () => {
+          const fetch = withHar(baseFetch);
+          const response = await fetch("https://postman-echo.com/post", {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/x-www-form-urlencoded"
+            },
+            body: "foo=1&bar=2&bar=three%20aka%203&baz=4"
+          });
+          expect(response.harEntry.request.bodySize).toBe(37);
+          expect(response.harEntry.request.postData).toEqual({
+            mimeType: "application/x-www-form-urlencoded",
+            params: [
+              { name: "foo", value: "1" },
+              { name: "bar", value: "2" },
+              { name: "bar", value: "three aka 3" },
+              { name: "baz", value: "4" }
+            ]
+          });
+        });
       });
 
       it("reports entries with the onHarEntry option", async () => {