From 9aeea627da426325ffe714cafa5e495c9750ef97 Mon Sep 17 00:00:00 2001 From: Vasu Raman Date: Sat, 28 May 2022 07:43:14 +0000 Subject: [PATCH 1/5] Have to add hook for exposing payload properly --- src/modules/http_observer.c | 54 +++++++++++++++++++++++++++++++++++++ src/mtev_http.c | 6 +++++ src/mtev_http.h | 5 ++++ src/mtev_http1.c | 1 + src/mtev_http2.c | 1 + 5 files changed, 67 insertions(+) diff --git a/src/modules/http_observer.c b/src/modules/http_observer.c index bf07c4c60..c2d842c90 100644 --- a/src/modules/http_observer.c +++ b/src/modules/http_observer.c @@ -64,11 +64,14 @@ typedef struct { uint64_t inbytes; uint64_t outbytes; mtev_hash_table info; + void *payload; + int64_t payload_length; } http_entry_t; static void http_entry_free(void *ve) { http_entry_t *e = ve; if(e == NULL) return; + free(e->payload); mtev_hash_destroy(&e->info, mtev_memory_safe_free, mtev_memory_safe_free); } @@ -94,6 +97,8 @@ allocate_entry(mtev_http_session_ctx *ctx) { newe->request_complete_ns = timeofday_nanos(); newe->id = ck_pr_faa_64(&global_id, 1); mtev_hash_init(&newe->info); + newe->payload = NULL; + newe->payload_length = 0; mtev_hash_replace(&lookup, (const char *)&newe->ctx, sizeof(newe->ctx), newe, NULL, mtev_memory_safe_free); return newe; } @@ -178,6 +183,51 @@ http_observer_prrp(void *closure, mtev_http_session_ctx *ctx) { return MTEV_HOOK_CONTINUE; } +static mtev_hook_return_t +http_observer_prpr(void *closure, mtev_http_request *req, const void *payload, int64_t payload_length) { + (void)closure; + if (payload_length <= 0) { + payload = NULL; + payload_length = 0; + } + const int64_t payload_limit = 1024*1024; + if (payload_length > payload_limit) { + payload_length = payload_limit; + } + char *payload_ptr = (char *)payload; + for (int64_t i = 0; i < payload_length; i++) + { + if (!isprint(payload_ptr++)) { + payload = NULL; + payload_length = 0; + break; + } + } + payload_ptr = NULL; + if (payload) { + payload_ptr = malloc(payload_length); + memcpy(payload_ptr, payload, payload_length); + } + mtev_memory_begin(); + + mtev_hash_iter iter = MTEV_HASH_ITER_ZERO; + while(mtev_hash_adv_spmc(&lookup, &iter)) { + mtev_http_session_ctx *ctx = (mtev_http_session_ctx *)iter.key.ptr; + if (req == mtev_http_session_request(ctx)) { + http_entry_t *entry = (http_entry_t *)iter.value.ptr; + if (entry->payload) { + free(entry->payload); + } + entry->payload = payload_ptr; + entry->payload_length = payload_ptr ? payload_length : 0; + http_entry_update(entry, ctx); + break; + } + } + mtev_memory_end(); + return MTEV_HOOK_CONTINUE; +} + static mtev_hook_return_t http_observer_rs(void *closure, mtev_http_session_ctx *ctx) { (void)closure; @@ -304,6 +354,9 @@ static void http_entry_json(mtev_http_session_ctx *ctx, http_entry_t *entry) { MJ_KV(o, "response_complete_offset_ns", MJ_INT64(entry->response_complete_ns - entry->request_start_ns)); MJ_KV(o, "received_bytes", MJ_INT64(entry->inbytes)); MJ_KV(o, "sent_bytes", MJ_INT64(entry->outbytes)); + if (entry->payload) { + MJ_KV(o, "payload_in", MJ_STRN(entry->payload, entry->payload_length)); + } mtev_hash_iter iter = MTEV_HASH_ITER_ZERO; while(mtev_hash_adv_spmc(&entry->info, &iter)) { MJ_KV(o, iter.key.str, MJ_STR(iter.value.str)); @@ -424,6 +477,7 @@ http_observer_driver_init(mtev_dso_generic_t *img) { http_response_send_hook_register("http_observer", http_observer_rs, NULL); http_request_log_hook_register("http_observer", http_observer_rl, NULL); http_post_request_read_payload_hook_register("http_observer", http_observer_prrp, NULL); + http_post_request_payload_retrieved_hook_register("http_observer", http_observer_prpr, NULL); mtev_rest_mountpoint_t *rule = mtev_http_rest_new_rule( "GET", "/module/http_observer/", "^requests.json$", requests_json_handler diff --git a/src/mtev_http.c b/src/mtev_http.c index 4fcf59bd4..60ada9699 100644 --- a/src/mtev_http.c +++ b/src/mtev_http.c @@ -68,6 +68,12 @@ MTEV_HOOK_IMPL(http_post_request_read_payload, (void *closure, mtev_http_session_ctx *ctx), (closure, ctx)) +MTEV_HOOK_IMPL(http_post_request_payload_retrieved, + (mtev_http_request *req, const void *payload, int64_t payload_length), + void *, closure, + (void *closure, mtev_http_request *req, const void *payload, int64_t payload_length), + (closure, req, payload, payload_length)) + MTEV_HOOK_IMPL(http_response_starting, (mtev_http_session_ctx *ctx), void *, closure, diff --git a/src/mtev_http.h b/src/mtev_http.h index 4b6ebedaa..a06f57694 100644 --- a/src/mtev_http.h +++ b/src/mtev_http.h @@ -279,6 +279,11 @@ MTEV_HOOK_PROTO(http_post_request_read_payload, void *, closure, (void *closure, mtev_http_session_ctx *ctx)) +MTEV_HOOK_PROTO(http_post_request_payload_retrieved, + (mtev_http_request *req, const void *payload, int64_t payload_length), + void *, closure, + (void *closure, mtev_http_request *req, const void *payload, int64_t payload_length)) + MTEV_HOOK_PROTO(http_response_starting, (mtev_http_session_ctx *ctx), void *, closure, diff --git a/src/mtev_http1.c b/src/mtev_http1.c index ea58f221d..e5fa07049 100644 --- a/src/mtev_http1.c +++ b/src/mtev_http1.c @@ -371,6 +371,7 @@ mtev_http1_request_set_upload(mtev_http1_request *req, const void * mtev_http1_request_get_upload(mtev_http1_request *req, int64_t *size) { if(size) *size = req->upload.size; + (void)http_post_request_payload_retrieved_hook_invoke((mtev_http_request *)req, req->upload.data, req->upload.size); return req->upload.data; } const char * diff --git a/src/mtev_http2.c b/src/mtev_http2.c index 90f18f3d1..85e63105b 100644 --- a/src/mtev_http2.c +++ b/src/mtev_http2.c @@ -402,6 +402,7 @@ mtev_http2_request_set_upload(mtev_http2_request *req, const void * mtev_http2_request_get_upload(mtev_http2_request *req, int64_t *size) { if(size) *size = req->upload.size; + (void)http_post_request_payload_retrieved_hook_invoke((mtev_http_request *)req, req->upload.data, req->upload.size); return req->upload.data; } const char * From 72888c8a63c438fee04bd3f5e0df380aa041e345 Mon Sep 17 00:00:00 2001 From: vraman Date: Tue, 31 May 2022 18:47:05 +0000 Subject: [PATCH 2/5] Fixed issues and tested with requests.json --- src/modules/http_observer.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/modules/http_observer.c b/src/modules/http_observer.c index c2d842c90..18b83883f 100644 --- a/src/modules/http_observer.c +++ b/src/modules/http_observer.c @@ -187,19 +187,19 @@ static mtev_hook_return_t http_observer_prpr(void *closure, mtev_http_request *req, const void *payload, int64_t payload_length) { (void)closure; if (payload_length <= 0) { - payload = NULL; - payload_length = 0; + payload = "(empty)"; + payload_length = 7; } const int64_t payload_limit = 1024*1024; if (payload_length > payload_limit) { payload_length = payload_limit; } char *payload_ptr = (char *)payload; - for (int64_t i = 0; i < payload_length; i++) + for (int64_t i = 0; i < payload_length; i++, payload_ptr++) { - if (!isprint(payload_ptr++)) { - payload = NULL; - payload_length = 0; + if (!isprint(*payload_ptr)) { + payload = "(binary)"; + payload_length = 8; break; } } @@ -208,13 +208,13 @@ http_observer_prpr(void *closure, mtev_http_request *req, const void *payload, i payload_ptr = malloc(payload_length); memcpy(payload_ptr, payload, payload_length); } - mtev_memory_begin(); + mtev_memory_begin(); mtev_hash_iter iter = MTEV_HASH_ITER_ZERO; while(mtev_hash_adv_spmc(&lookup, &iter)) { - mtev_http_session_ctx *ctx = (mtev_http_session_ctx *)iter.key.ptr; + http_entry_t *entry = (http_entry_t *)iter.value.ptr; + mtev_http_session_ctx *ctx = entry->ctx; if (req == mtev_http_session_request(ctx)) { - http_entry_t *entry = (http_entry_t *)iter.value.ptr; if (entry->payload) { free(entry->payload); } From 5db5c1da4068c879b8abebb43003de6afefac7ce Mon Sep 17 00:00:00 2001 From: vraman Date: Tue, 31 May 2022 21:11:52 +0000 Subject: [PATCH 3/5] Debug check, to be removed --- src/backtrace-support/mtev-http-observer-module.lua | 2 ++ src/modules/http_observer.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/backtrace-support/mtev-http-observer-module.lua b/src/backtrace-support/mtev-http-observer-module.lua index 1f86f9526..d44aec439 100644 --- a/src/backtrace-support/mtev-http-observer-module.lua +++ b/src/backtrace-support/mtev-http-observer-module.lua @@ -118,6 +118,8 @@ end -- uint64_t inbytes; -- uint64_t outbytes; -- mtev_hash_table info; 128 +-- void *payload; +-- int64_t payload_length; -- } http_entry_t; -- local function walk_hash(a_hash) diff --git a/src/modules/http_observer.c b/src/modules/http_observer.c index 18b83883f..1e9739ea0 100644 --- a/src/modules/http_observer.c +++ b/src/modules/http_observer.c @@ -97,6 +97,8 @@ allocate_entry(mtev_http_session_ctx *ctx) { newe->request_complete_ns = timeofday_nanos(); newe->id = ck_pr_faa_64(&global_id, 1); mtev_hash_init(&newe->info); +// VASU DEBUG +mtevL(mtev_error, "Entry %p, info %p, payload %p, length %p\n", newe, &newe->info, &newe->payload, &newe->payload_length); newe->payload = NULL; newe->payload_length = 0; mtev_hash_replace(&lookup, (const char *)&newe->ctx, sizeof(newe->ctx), newe, NULL, mtev_memory_safe_free); From 29a248fa9a84d341faacacd5bc733d10450a8a4a Mon Sep 17 00:00:00 2001 From: vraman Date: Tue, 31 May 2022 23:24:51 +0000 Subject: [PATCH 4/5] Make payload sizefor http observer configuration --- src/backtrace-support/mtev-http-observer-module.lua | 4 ++-- src/modules/http_observer.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backtrace-support/mtev-http-observer-module.lua b/src/backtrace-support/mtev-http-observer-module.lua index d44aec439..368732d58 100644 --- a/src/backtrace-support/mtev-http-observer-module.lua +++ b/src/backtrace-support/mtev-http-observer-module.lua @@ -117,8 +117,8 @@ end -- uint64_t response_complete_ns; -- uint64_t inbytes; -- uint64_t outbytes; --- mtev_hash_table info; 128 --- void *payload; +-- mtev_hash_table info; 128 (vasu got 80d (0x50) for this) +-- void *payload; 144d (0x90) -- int64_t payload_length; -- } http_entry_t; -- diff --git a/src/modules/http_observer.c b/src/modules/http_observer.c index 1e9739ea0..863560580 100644 --- a/src/modules/http_observer.c +++ b/src/modules/http_observer.c @@ -42,7 +42,7 @@ #include static mtev_log_stream_t debugls, errorls; -static uint32_t max_count = 10000, max_age = 30; +static uint32_t max_count = 10000, max_age = 30, max_payload = 256 * 1024; static uint64_t global_id; static mtev_hash_table lookup, hdrin_extract, hdrout_extract; @@ -97,8 +97,6 @@ allocate_entry(mtev_http_session_ctx *ctx) { newe->request_complete_ns = timeofday_nanos(); newe->id = ck_pr_faa_64(&global_id, 1); mtev_hash_init(&newe->info); -// VASU DEBUG -mtevL(mtev_error, "Entry %p, info %p, payload %p, length %p\n", newe, &newe->info, &newe->payload, &newe->payload_length); newe->payload = NULL; newe->payload_length = 0; mtev_hash_replace(&lookup, (const char *)&newe->ctx, sizeof(newe->ctx), newe, NULL, mtev_memory_safe_free); @@ -192,9 +190,8 @@ http_observer_prpr(void *closure, mtev_http_request *req, const void *payload, i payload = "(empty)"; payload_length = 7; } - const int64_t payload_limit = 1024*1024; - if (payload_length > payload_limit) { - payload_length = payload_limit; + if (payload_length > (int64_t)max_payload) { + payload_length = (int64_t)max_payload; } char *payload_ptr = (char *)payload; for (int64_t i = 0; i < payload_length; i++, payload_ptr++) @@ -456,6 +453,9 @@ http_observer_driver_config(mtev_dso_generic_t *img, mtev_hash_table *options) { if(mtev_hash_retr_str(options, "max_age", strlen("max_age"), &vstr)) { max_age = atoi(vstr); } + if(mtev_hash_retr_str(options, "max_payload", strlen("max_payload"), &vstr)) { + max_payload = atoi(vstr); + } mtev_hash_iter iter = MTEV_HASH_ITER_ZERO; while(mtev_hash_adv(options, &iter)) { if(!strncmp(iter.key.str, "header_in_", 10)) { From bc7fc55542d5e9cc937e1e05b6dc6f791b1a05e5 Mon Sep 17 00:00:00 2001 From: Vasu Raman Date: Wed, 1 Jun 2022 04:58:30 +0000 Subject: [PATCH 5/5] Add payload to backtrace k-v --- src/backtrace-support/mtev-http1-module.lua | 10 ++++++++-- src/backtrace-support/mtev-http2-module.lua | 12 +++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/backtrace-support/mtev-http1-module.lua b/src/backtrace-support/mtev-http1-module.lua index 3992e6201..8c6b36c1e 100644 --- a/src/backtrace-support/mtev-http1-module.lua +++ b/src/backtrace-support/mtev-http1-module.lua @@ -2,9 +2,10 @@ function L(...) pmodule.log(pmodule.log_level.info, string.format(...)) end -local function bt2str(bt_var) +local function bt2str(bt_var, len) + len = len or 256 local addr = pmodule.variable_from_bt(bt_var):value() - return pmodule.address_read_string(addr, 256) + return pmodule.address_read_string(addr, len) end local function bt2val(bt_var) @@ -20,12 +21,17 @@ local function variable_http1_cb(pt_var, bt_var) local qs = bt2str(bt_var.orig_qs) qs = qs and ("?" .. qs) or "" local length = bt2val(bt_var.content_length) or -1 + local payload_len = bt2val(bt_var.upload.size) + local payload = bt2str(bt_var.upload.data, payload_len + 1) pt_var:thread():annotate( pmodule.annotation.comment, string.format("mtev_http1_request: %s %s%s (%d)", method, uri, qs, length)) pt_var:backtrace():add_kv_string( "mtev_http_request", string.format("%s %s%s (%d)", method, uri, qs, length)) + pt_var:backtrace():add_kv_string( + "mtev_http_payload_in", + string.format("%s", payload)) end function pm_mtev_http1_req() diff --git a/src/backtrace-support/mtev-http2-module.lua b/src/backtrace-support/mtev-http2-module.lua index aea3d65ab..4f4c5ad50 100644 --- a/src/backtrace-support/mtev-http2-module.lua +++ b/src/backtrace-support/mtev-http2-module.lua @@ -2,9 +2,10 @@ local function L(...) pmodule.log(pmodule.log_level.info, string.format(...)) end -local function bt2str(bt_var) +local function bt2str(bt_var, len) + len = len or 256 local addr = pmodule.variable_from_bt(bt_var):value() - return pmodule.address_read_string(addr, 256) + return pmodule.address_read_string(addr, len) end local function bt2val(bt_var) @@ -18,13 +19,18 @@ local function variable_http2_cb(pt_var, bt_var) local qs = bt2str(bt_var.orig_qs) qs = qs and ("?" .. qs) or "" local length = bt2val(bt_var.content_length) or -1 + local payload_len = bt2val(bt_var.upload.size) + local payload = bt2str(bt_var.upload.data, payload_len + 1) pt_var:thread():annotate( pmodule.annotation.comment, string.format("mtev_http2_request: %s %s%s (%d)", method, uri, qs, length)) pt_var:backtrace():add_kv_string( "mtev_http_request", string.format("%s %s%s (%d)", method, uri, qs, length)) -end + pt_var:backtrace():add_kv_string( + "mtev_http_payload_in", + string.format("%s", payload)) + end local function pm_mtev_http2_req() L("module-mtev-http2: load")