Skip to content

Commit 40239a4

Browse files
kingthorinkingthorin
authored
Merge pull request zaproxy#9002 from psiinon/tree/struct-data
Fixed structured POST data node names
2 parents e7162a9 + 2eb501d commit 40239a4

6 files changed

Lines changed: 813 additions & 91 deletions

File tree

‎zap/src/main/java/org/zaproxy/zap/model/SessionStructure.java‎

Lines changed: 167 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,16 @@
2323
import java.util.List;
2424
import java.util.Locale;
2525
import java.util.Objects;
26+
import java.util.stream.Collectors;
27+
import net.sf.json.JSONException;
2628
import org.apache.commons.httpclient.URI;
2729
import org.apache.commons.httpclient.URIException;
30+
import org.apache.commons.lang3.Strings;
2831
import org.apache.logging.log4j.LogManager;
2932
import org.apache.logging.log4j.Logger;
3033
import org.parosproxy.paros.Constant;
3134
import org.parosproxy.paros.core.scanner.Variant;
35+
import org.parosproxy.paros.core.scanner.VariantMultipartFormParameters;
3236
import org.parosproxy.paros.db.DatabaseException;
3337
import org.parosproxy.paros.db.RecordStructure;
3438
import org.parosproxy.paros.model.HistoryReference;
@@ -40,16 +44,18 @@
4044
import org.parosproxy.paros.network.HttpMalformedHeaderException;
4145
import org.parosproxy.paros.network.HttpMessage;
4246
import org.parosproxy.paros.network.HttpRequestHeader;
47+
import org.zaproxy.zap.utils.JsonUtil;
48+
import org.zaproxy.zap.utils.XmlUtils;
4349

4450
public class SessionStructure {
4551

4652
public static final String ROOT = "Root";
4753

54+
public static final int MAX_NODE_NAME_SIZE = 512;
55+
4856
public static final String DATA_DRIVEN_NODE_PREFIX = "\u00AB";
4957
public static final String DATA_DRIVEN_NODE_POSTFIX = "\u00BB";
5058
public static final String DATA_DRIVEN_NODE_REGEX = "(.+?)";
51-
private static final String MULTIPART_FORM_DATA_DISPLAY =
52-
"(" + HttpHeader.FORM_MULTIPART_CONTENT_TYPE + ")";
5359

5460
private static final Logger LOGGER = LogManager.getLogger(SessionStructure.class);
5561

@@ -200,10 +206,16 @@ private static String getNodeName(
200206
String host = getHostName(uri);
201207
String nodeUrl = pathsToUrl(host, paths, paths.size());
202208

203-
String params = getParams(session, uri, postData, contentType);
204-
if (params.length() > 0) {
205-
nodeUrl += " " + params;
209+
try {
210+
HttpMessage msg = getMsg(uri, method, postData, contentType);
211+
String params = getParams(session, msg);
212+
if (!params.isEmpty()) {
213+
nodeUrl += " " + params;
214+
}
215+
} catch (HttpMalformedHeaderException e) {
216+
LOGGER.error(e.getMessage(), e);
206217
}
218+
207219
return nodeUrl;
208220
}
209221

@@ -214,7 +226,7 @@ private static String getNodeName(
214226

215227
if (msg != null) {
216228
String params = getParams(session, msg);
217-
if (params.length() > 0) {
229+
if (!params.isEmpty()) {
218230
nodeUrl = nodeUrl + " " + params;
219231
}
220232
}
@@ -231,12 +243,16 @@ private static String getNodeName(
231243
* @since 2.10.0
232244
*/
233245
public static String getNodeName(Model model, HttpMessage msg) throws URIException {
234-
return getNodeName(
235-
model,
236-
msg.getRequestHeader().getURI(),
237-
msg.getRequestHeader().getMethod(),
238-
msg.getRequestBody().toString(),
239-
msg.getRequestHeader().getHeader(HttpHeader.CONTENT_TYPE));
246+
Session session = model.getSession();
247+
URI uri = msg.getRequestHeader().getURI();
248+
List<String> paths = getTreePath(model, uri);
249+
String host = getHostName(uri);
250+
String nodeUrl = pathsToUrl(host, paths, paths.size());
251+
String params = getParams(session, msg);
252+
if (!params.isEmpty()) {
253+
nodeUrl += " " + params;
254+
}
255+
return nodeUrl;
240256
}
241257

242258
public static String getLeafName(Model model, String nodeName, HttpMessage msg) {
@@ -256,7 +272,8 @@ public static String getLeafName(Model model, String nodeName, HttpMessage msg)
256272
convertNVP(
257273
model.getSession().getParameters(msg, Type.url),
258274
org.parosproxy.paros.core.scanner.NameValuePair.TYPE_QUERY_STRING);
259-
if (msg.getRequestHeader().getMethod().equalsIgnoreCase(HttpRequestHeader.POST)) {
275+
276+
if (msg.getRequestBody().length() > 0) {
260277
params.addAll(
261278
convertNVP(
262279
model.getSession().getParameters(msg, Type.form),
@@ -284,13 +301,7 @@ public static String getLeafName(
284301
throws HttpMalformedHeaderException {
285302
Objects.requireNonNull(uri);
286303
Objects.requireNonNull(method);
287-
HttpMessage msg = new HttpMessage(uri);
288-
msg.getRequestHeader().setMethod(method);
289-
if (method.equalsIgnoreCase(HttpRequestHeader.POST)) {
290-
msg.getRequestBody().setBody(postData);
291-
msg.getRequestHeader().setContentLength(msg.getRequestBody().length());
292-
}
293-
return getLeafName(model, nodeName, msg);
304+
return getLeafName(model, nodeName, getMsg(uri, method, postData, null));
294305
}
295306

296307
public static String getLeafName(
@@ -303,43 +314,18 @@ public static String getLeafName(
303314
sb.append(":");
304315
sb.append(nodeName);
305316

306-
if (method.equalsIgnoreCase(HttpRequestHeader.POST)) {
307-
sb.append(
308-
getQueryParamString(
309-
convertParosNVP(
310-
params,
311-
org.parosproxy.paros.core.scanner.NameValuePair
312-
.TYPE_QUERY_STRING),
313-
true));
314-
315-
String contentType = message.getRequestHeader().getHeader(HttpHeader.CONTENT_TYPE);
316-
if (contentType != null
317-
&& contentType.startsWith(HttpHeader.FORM_MULTIPART_CONTENT_TYPE)) {
318-
sb.append(MULTIPART_FORM_DATA_DISPLAY);
319-
} else {
320-
sb.append(
321-
getQueryParamString(
322-
convertParosNVP(
323-
params,
324-
org.parosproxy.paros.core.scanner.NameValuePair
325-
.TYPE_POST_DATA),
326-
false));
327-
}
328-
} else {
329-
sb.append(
330-
getQueryParamString(
331-
convertParosNVP(
332-
params,
333-
org.parosproxy.paros.core.scanner.NameValuePair
334-
.TYPE_QUERY_STRING),
335-
false));
336-
sb.append(
337-
getQueryParamString(
338-
convertParosNVP(
339-
params,
340-
org.parosproxy.paros.core.scanner.NameValuePair.TYPE_POST_DATA),
341-
false));
342-
}
317+
List<NameValuePair> postParams =
318+
convertParosNVP(
319+
params, org.parosproxy.paros.core.scanner.NameValuePair.TYPE_POST_DATA);
320+
321+
sb.append(
322+
getQueryParamString(
323+
convertParosNVP(
324+
params,
325+
org.parosproxy.paros.core.scanner.NameValuePair.TYPE_QUERY_STRING),
326+
!postParams.isEmpty()));
327+
328+
sb.append(getPostParamString(message, getQueryParamString(postParams, false)));
343329

344330
return sb.toString();
345331
}
@@ -499,7 +485,6 @@ private static RecordStructure addStructure(
499485
int historyId,
500486
boolean newOnly)
501487
throws DatabaseException, URIException {
502-
// String nodeUrl = pathsToUrl(host, paths, size);
503488
Session session = model.getSession();
504489
String nodeName = getNodeName(session, host, msg, paths, size);
505490
String parentName = pathsToUrl(host, paths, size - 1);
@@ -508,7 +493,7 @@ private static RecordStructure addStructure(
508493
if (msg != null) {
509494
url = msg.getRequestHeader().getURI().toString();
510495
String params = getParams(session, msg);
511-
if (params.length() > 0) {
496+
if (!params.isEmpty()) {
512497
nodeName = nodeName + " " + params;
513498
}
514499
}
@@ -649,27 +634,125 @@ public static StructuralNode getRootNode(Model model) {
649634
}
650635

651636
private static String getParams(Session session, HttpMessage msg) throws URIException {
652-
return getParams(
653-
session,
654-
msg.getRequestHeader().getURI(),
655-
msg.getRequestBody().toString(),
656-
msg.getRequestHeader().getHeader(HttpHeader.CONTENT_TYPE));
657-
}
637+
String contentType = msg.getRequestHeader().getHeader(HttpHeader.CONTENT_TYPE);
638+
String reqBody = msg.getRequestBody().toString();
639+
boolean hasReqBody = contentType != null && !reqBody.isEmpty();
658640

659-
private static String getParams(
660-
Session session, URI uri, String requestBody, String contentType) throws URIException {
661-
boolean hasReqBody = contentType != null && requestBody != null && !requestBody.isEmpty();
662-
String leafParams = getQueryParamString(session.getUrlParameters(uri), hasReqBody);
641+
String leafParams =
642+
getQueryParamString(
643+
session.getUrlParameters(msg.getRequestHeader().getURI()), hasReqBody);
663644
if (!hasReqBody) {
664645
return leafParams;
665646
}
666647

648+
return leafParams
649+
+ getPostParamString(
650+
msg,
651+
getQueryParamString(
652+
session.getFormParameters(msg.getRequestHeader().getURI(), reqBody),
653+
false));
654+
}
655+
656+
private static String getPostParamString(HttpMessage msg, String fallback) {
657+
String contentType = msg.getRequestHeader().getHeader(HttpHeader.CONTENT_TYPE);
658+
String reqBody = msg.getRequestBody().toString();
659+
if (!reqBody.isEmpty()) {
660+
String str;
661+
if (contentType != null) {
662+
str = getPostParamStringForContentType(msg, contentType, reqBody);
663+
} else {
664+
str = guessPostParamString(msg, reqBody);
665+
}
666+
if (str != null) {
667+
return "(" + str + ")";
668+
}
669+
}
670+
return fallback;
671+
}
672+
673+
private static String getPostParamStringForContentType(
674+
HttpMessage msg, String contentType, String body) {
667675
if (contentType.startsWith(HttpHeader.FORM_MULTIPART_CONTENT_TYPE)) {
668-
leafParams += MULTIPART_FORM_DATA_DISPLAY;
669-
} else if (contentType.startsWith("application/x-www-form-urlencoded")) {
670-
leafParams += getQueryParamString(session.getFormParameters(uri, requestBody), false);
676+
VariantMultipartFormParameters mfp = new VariantMultipartFormParameters();
677+
mfp.setMessage(msg);
678+
return "multipart:"
679+
+ getNameList(
680+
mfp.getParamList().stream()
681+
.filter(p -> isRelevantMultipartParam(p.getType()))
682+
.toList());
683+
}
684+
if (msg.getRequestHeader().hasContentType("json")) {
685+
try {
686+
return JsonUtil.getJsonKeyString(body);
687+
} catch (JSONException e) {
688+
return body.substring(0, Math.min(body.length(), MAX_NODE_NAME_SIZE));
689+
}
690+
}
691+
if (msg.getRequestHeader().hasContentType("xml")) {
692+
try {
693+
return XmlUtils.getXmlKeyString(body);
694+
} catch (Exception e) {
695+
return body.substring(0, Math.min(body.length(), MAX_NODE_NAME_SIZE));
696+
}
671697
}
672-
return leafParams;
698+
return null;
699+
}
700+
701+
private static boolean isRelevantMultipartParam(int type) {
702+
return type == org.parosproxy.paros.core.scanner.NameValuePair.TYPE_MULTIPART_DATA_FILE_NAME
703+
|| type
704+
== org.parosproxy.paros.core.scanner.NameValuePair
705+
.TYPE_MULTIPART_DATA_PARAM;
706+
}
707+
708+
private static String getNameList(List<org.parosproxy.paros.core.scanner.NameValuePair> nvp) {
709+
return nvp.stream()
710+
.map(org.parosproxy.paros.core.scanner.NameValuePair::getName)
711+
.collect(Collectors.joining(","));
712+
}
713+
714+
/** Try to work out the post data param string where we have no content type. */
715+
private static String guessPostParamString(HttpMessage msg, String body) {
716+
try {
717+
String str = JsonUtil.getJsonKeyString(body);
718+
if (!str.isEmpty()) {
719+
return str;
720+
}
721+
} catch (Exception e) {
722+
// Ignore
723+
}
724+
try {
725+
String str = XmlUtils.getXmlKeyString(msg.getRequestBody().toString());
726+
if (!str.isEmpty()) {
727+
return str;
728+
}
729+
} catch (Exception e) {
730+
// Ignore
731+
}
732+
try {
733+
if (Strings.CI.contains(body, "Content-Disposition")) {
734+
String[] bodyLines = body.split(HttpHeader.CRLF);
735+
if (bodyLines.length > 2 && bodyLines[0].startsWith("--")) {
736+
// Looking likely, we need to reform the content type
737+
msg.getRequestHeader()
738+
.setHeader(
739+
HttpHeader.CONTENT_TYPE,
740+
HttpHeader.FORM_MULTIPART_CONTENT_TYPE
741+
+ "; boundary="
742+
+ bodyLines[0].substring(2));
743+
VariantMultipartFormParameters mfp = new VariantMultipartFormParameters();
744+
745+
mfp.setMessage(msg);
746+
String str = getNameList(mfp.getParamList());
747+
if (!str.isEmpty()) {
748+
return "multipart:" + str;
749+
}
750+
}
751+
}
752+
} catch (Exception e) {
753+
// Ignore
754+
}
755+
return null;
673756
}
674757

675758
private static String getQueryParamString(List<NameValuePair> list, boolean isUrlWithPostData) {
@@ -697,4 +780,15 @@ private static String getQueryParamString(List<NameValuePair> list, boolean isUr
697780

698781
return result;
699782
}
783+
784+
private static HttpMessage getMsg(URI uri, String method, String postData, String contentType)
785+
throws HttpMalformedHeaderException {
786+
HttpMessage msg = new HttpMessage(uri);
787+
msg.getRequestHeader().setMethod(method);
788+
if (contentType != null) {
789+
msg.getRequestHeader().setHeader(HttpHeader.CONTENT_TYPE, contentType);
790+
}
791+
msg.getRequestBody().setBody(postData);
792+
return msg;
793+
}
700794
}

0 commit comments

Comments
 (0)