Skip to content

Commit 31af5c0

Browse files
dogancanbakirdogancanbakir
authored
Merge pull request #2462 from projectdiscovery/2418-store-response-only-matched
fixing output writing
2 parents a4a5b2e + 98e6af0 commit 31af5c0

3 files changed

Lines changed: 223 additions & 44 deletions

File tree

‎runner/options.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ type Options struct {
343343
HeadlessOptionalArguments goflags.StringSlice
344344
Protocol string
345345
OutputFilterErrorPagePath string
346-
DisableStdout bool
346+
DisableStdout bool
347347

348348
JavascriptCodes goflags.StringSlice
349349

@@ -696,6 +696,30 @@ func ParseOptions() *Options {
696696
return options
697697
}
698698

699+
func (options *Options) HasMatcherOrFilter() bool {
700+
return len(options.matchStatusCode) > 0 ||
701+
len(options.matchContentLength) > 0 ||
702+
len(options.filterStatusCode) > 0 ||
703+
len(options.filterContentLength) > 0 ||
704+
len(options.matchRegexes) > 0 ||
705+
len(options.filterRegexes) > 0 ||
706+
len(options.matchLinesCount) > 0 ||
707+
len(options.matchWordsCount) > 0 ||
708+
len(options.filterLinesCount) > 0 ||
709+
len(options.filterWordsCount) > 0 ||
710+
len(options.OutputMatchString) > 0 ||
711+
len(options.OutputFilterString) > 0 ||
712+
len(options.OutputMatchFavicon) > 0 ||
713+
len(options.OutputFilterFavicon) > 0 ||
714+
len(options.OutputMatchCdn) > 0 ||
715+
len(options.OutputFilterCdn) > 0 ||
716+
len(options.OutputFilterPageType) > 0 ||
717+
options.OutputMatchCondition != "" ||
718+
options.OutputFilterCondition != "" ||
719+
options.OutputMatchResponseTime != "" ||
720+
options.OutputFilterResponseTime != ""
721+
}
722+
699723
func (options *Options) ValidateOptions() error {
700724
if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
701725
return fmt.Errorf("file '%s' does not exist", options.InputFile)

‎runner/runner.go‎

Lines changed: 46 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2458,53 +2458,56 @@ retry:
24582458
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
24592459

24602460
var responsePath, fileNameHash string
2461-
// store response
2461+
// store response — when matchers/filters are active, defer writing to the
2462+
// output loop so only matched responses are persisted to disk.
24622463
if scanopts.StoreResponse || scanopts.StoreChain {
2463-
if r.options.OmitBody {
2464-
resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "")
2465-
}
2466-
responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
2467-
// URL.EscapedString returns that can be used as filename
2468-
respRaw := resp.Raw
2469-
reqRaw := requestDump
2470-
if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
2471-
respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
2472-
}
2473-
data := reqRaw
2474-
if scanopts.StoreChain && resp.HasChain() {
2475-
data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...)
2476-
}
2477-
data = append(data, respRaw...)
2478-
data = append(data, []byte("\n\n\n")...)
2479-
data = append(data, []byte(fullURL)...)
2480-
_ = fileutil.CreateFolder(responseBaseDir)
2481-
2482-
basePath := strings.TrimSuffix(responsePath, ".txt")
2483-
var idx int
2484-
for idx = 0; ; idx++ {
2485-
targetPath := responsePath
2486-
if idx > 0 {
2487-
targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
2488-
}
2489-
f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
2490-
if err == nil {
2491-
_, writeErr := f.Write(data)
2492-
_ = f.Close()
2493-
if writeErr != nil {
2494-
gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
2464+
fileNameHash = hash
2465+
2466+
if !r.options.HasMatcherOrFilter() {
2467+
if r.options.OmitBody {
2468+
resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "")
2469+
}
2470+
responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
2471+
// URL.EscapedString returns that can be used as filename
2472+
respRaw := resp.Raw
2473+
reqRaw := requestDump
2474+
if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
2475+
respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
2476+
}
2477+
data := reqRaw
2478+
if scanopts.StoreChain && resp.HasChain() {
2479+
data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...)
2480+
}
2481+
data = append(data, respRaw...)
2482+
data = append(data, []byte("\n\n\n")...)
2483+
data = append(data, []byte(fullURL)...)
2484+
_ = fileutil.CreateFolder(responseBaseDir)
2485+
2486+
basePath := strings.TrimSuffix(responsePath, ".txt")
2487+
var idx int
2488+
for idx = 0; ; idx++ {
2489+
targetPath := responsePath
2490+
if idx > 0 {
2491+
targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
2492+
}
2493+
f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
2494+
if err == nil {
2495+
_, writeErr := f.Write(data)
2496+
_ = f.Close()
2497+
if writeErr != nil {
2498+
gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
2499+
}
2500+
break
2501+
}
2502+
if !os.IsExist(err) {
2503+
gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
2504+
break
24952505
}
2496-
break
2497-
}
2498-
if !os.IsExist(err) {
2499-
gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
2500-
break
25012506
}
2502-
}
25032507

2504-
if idx == 0 {
2505-
fileNameHash = hash
2506-
} else {
2507-
fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
2508+
if idx > 0 {
2509+
fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
2510+
}
25082511
}
25092512
}
25102513

‎runner/runner_test.go‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,158 @@ func TestRunner_testAndSet_concurrent(t *testing.T) {
386386
require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key")
387387
}
388388

389+
func TestOptions_hasMatcherOrFilter(t *testing.T) {
390+
tests := []struct {
391+
name string
392+
options Options
393+
expected bool
394+
}{
395+
{
396+
name: "no matchers or filters",
397+
options: Options{},
398+
expected: false,
399+
},
400+
{
401+
name: "match status code",
402+
options: Options{OutputMatchStatusCode: "200"},
403+
expected: true,
404+
},
405+
{
406+
name: "filter status code",
407+
options: Options{OutputFilterStatusCode: "403,401"},
408+
expected: true,
409+
},
410+
{
411+
name: "match string",
412+
options: Options{OutputMatchString: []string{"admin"}},
413+
expected: true,
414+
},
415+
{
416+
name: "filter string",
417+
options: Options{OutputFilterString: []string{"error"}},
418+
expected: true,
419+
},
420+
{
421+
name: "match content length",
422+
options: Options{OutputMatchContentLength: "100"},
423+
expected: true,
424+
},
425+
{
426+
name: "filter content length",
427+
options: Options{OutputFilterContentLength: "0"},
428+
expected: true,
429+
},
430+
{
431+
name: "match regex",
432+
options: Options{OutputMatchRegex: []string{"admin.*panel"}},
433+
expected: true,
434+
},
435+
{
436+
name: "filter regex",
437+
options: Options{OutputFilterRegex: []string{"error"}},
438+
expected: true,
439+
},
440+
{
441+
name: "match lines count",
442+
options: Options{OutputMatchLinesCount: "50"},
443+
expected: true,
444+
},
445+
{
446+
name: "filter lines count",
447+
options: Options{OutputFilterLinesCount: "0"},
448+
expected: true,
449+
},
450+
{
451+
name: "match words count",
452+
options: Options{OutputMatchWordsCount: "100"},
453+
expected: true,
454+
},
455+
{
456+
name: "filter words count",
457+
options: Options{OutputFilterWordsCount: "0"},
458+
expected: true,
459+
},
460+
{
461+
name: "match favicon",
462+
options: Options{OutputMatchFavicon: []string{"1494302000"}},
463+
expected: true,
464+
},
465+
{
466+
name: "filter favicon",
467+
options: Options{OutputFilterFavicon: []string{"1494302000"}},
468+
expected: true,
469+
},
470+
{
471+
name: "match cdn",
472+
options: Options{OutputMatchCdn: []string{"cloudflare"}},
473+
expected: true,
474+
},
475+
{
476+
name: "filter cdn",
477+
options: Options{OutputFilterCdn: []string{"cloudflare"}},
478+
expected: true,
479+
},
480+
{
481+
name: "match condition",
482+
options: Options{OutputMatchCondition: "status_code == 200"},
483+
expected: true,
484+
},
485+
{
486+
name: "filter condition",
487+
options: Options{OutputFilterCondition: "status_code == 403"},
488+
expected: true,
489+
},
490+
{
491+
name: "match response time",
492+
options: Options{OutputMatchResponseTime: "< 1"},
493+
expected: true,
494+
},
495+
{
496+
name: "filter response time",
497+
options: Options{OutputFilterResponseTime: "> 5"},
498+
expected: true,
499+
},
500+
{
501+
name: "filter page type",
502+
options: Options{OutputFilterPageType: []string{"error"}},
503+
expected: true,
504+
},
505+
}
506+
507+
for _, tc := range tests {
508+
t.Run(tc.name, func(t *testing.T) {
509+
opts := tc.options
510+
err := opts.ValidateOptions()
511+
require.Nil(t, err)
512+
require.Equal(t, tc.expected, opts.HasMatcherOrFilter(),
513+
"HasMatcherOrFilter() should be %v for %s", tc.expected, tc.name)
514+
})
515+
}
516+
}
517+
518+
func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
519+
dir := t.TempDir()
520+
opts := &Options{
521+
StoreResponse: true,
522+
StoreResponseDir: dir,
523+
}
524+
err := opts.ValidateOptions()
525+
require.Nil(t, err)
526+
require.False(t, opts.HasMatcherOrFilter())
527+
}
528+
529+
func TestStoreResponse_withMatcherSetsFlag(t *testing.T) {
530+
dir := t.TempDir()
531+
opts := &Options{
532+
StoreResponse: true,
533+
StoreResponseDir: dir,
534+
OutputMatchStatusCode: "200",
535+
}
536+
err := opts.ValidateOptions()
537+
require.Nil(t, err)
538+
require.True(t, opts.HasMatcherOrFilter())
539+
}
540+
389541
func TestRunner_duplicate(t *testing.T) {
390542
const (
391543
pageA = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here</body></html>"

0 commit comments

Comments
 (0)