Skip to content

Commit fb1ea1c

Browse files
Mzack9999Mzack9999
committed
Merge branch 'dev' into feature/secret-file-auth
2 parents a4ae407 + f6aa159 commit fb1ea1c

9 files changed

Lines changed: 402 additions & 18 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,10 @@ Usage:
9292

9393
Flags:
9494
INPUT:
95-
-l, -list string input file containing list of hosts to process
96-
-rr, -request string file containing raw request
97-
-u, -target string[] input target host(s) to probe
95+
-l, -list string input file containing list of hosts to process
96+
-rr, -request string file containing raw request
97+
-u, -target string[] input target host(s) to probe
98+
-im, -input-mode string mode of input file (burp)
9899

99100
PROBES:
100101
-sc, -status-code display response status-code
@@ -279,6 +280,7 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
279280
# Notes
280281

281282
- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
283+
- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
282284
- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
283285
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
284286
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)

‎common/inputformats/burp.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package inputformats
2+
3+
import (
4+
"io"
5+
6+
"github.com/pkg/errors"
7+
"github.com/projectdiscovery/gologger"
8+
"github.com/seh-msft/burpxml"
9+
)
10+
11+
// BurpFormat is a Burp Suite XML file parser
12+
type BurpFormat struct{}
13+
14+
// NewBurpFormat creates a new Burp XML file parser
15+
func NewBurpFormat() *BurpFormat {
16+
return &BurpFormat{}
17+
}
18+
19+
var _ Format = &BurpFormat{}
20+
21+
// Name returns the name of the format
22+
func (b *BurpFormat) Name() string {
23+
return "burp"
24+
}
25+
26+
// Parse parses the Burp XML input and calls the provided callback
27+
// function for each URL it discovers.
28+
func (b *BurpFormat) Parse(input io.Reader, callback func(url string) bool) error {
29+
items, err := burpxml.Parse(input, true)
30+
if err != nil {
31+
return errors.Wrap(err, "could not parse burp xml")
32+
}
33+
34+
for i, item := range items.Items {
35+
if item.Url == "" {
36+
gologger.Debug().Msgf("Skipping burp item %d: empty URL", i)
37+
continue
38+
}
39+
if !callback(item.Url) {
40+
break
41+
}
42+
}
43+
return nil
44+
}

‎common/inputformats/burp_test.go‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
package inputformats
2+
3+
import (
4+
"strings"
5+
"testing"
6+
)
7+
8+
func TestBurpFormat_Name(t *testing.T) {
9+
b := NewBurpFormat()
10+
if b.Name() != "burp" {
11+
t.Errorf("Expected name 'burp', got '%s'", b.Name())
12+
}
13+
}
14+
15+
func TestBurpFormat_Parse(t *testing.T) {
16+
burpXML := `<?xml version="1.0"?>
17+
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
18+
<item>
19+
<time>Sat Sep 30 20:11:32 IST 2023</time>
20+
<url><![CDATA[http://example.com/path1]]></url>
21+
<host ip="127.0.0.1">example.com</host>
22+
<port>80</port>
23+
<protocol>http</protocol>
24+
<method><![CDATA[GET]]></method>
25+
<path><![CDATA[/path1]]></path>
26+
<extension>null</extension>
27+
<request base64="true"><![CDATA[R0VUIC8gSFRUUC8xLjE=]]></request>
28+
<status>200</status>
29+
<responselength>100</responselength>
30+
<mimetype>HTML</mimetype>
31+
<response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
32+
<comment></comment>
33+
</item>
34+
<item>
35+
<time>Sat Sep 30 20:08:54 IST 2023</time>
36+
<url><![CDATA[https://example.com/path2]]></url>
37+
<host ip="127.0.0.1">example.com</host>
38+
<port>443</port>
39+
<protocol>https</protocol>
40+
<method><![CDATA[POST]]></method>
41+
<path><![CDATA[/path2]]></path>
42+
<extension>null</extension>
43+
<request base64="true"><![CDATA[UE9TVCAvIEhUVFAvMS4x]]></request>
44+
<status>200</status>
45+
<responselength>100</responselength>
46+
<mimetype>JSON</mimetype>
47+
<response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
48+
<comment></comment>
49+
</item>
50+
</items>`
51+
52+
b := NewBurpFormat()
53+
var urls []string
54+
55+
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
56+
urls = append(urls, url)
57+
return true
58+
})
59+
60+
if err != nil {
61+
t.Fatalf("Parse returned error: %v", err)
62+
}
63+
64+
if len(urls) != 2 {
65+
t.Errorf("Expected 2 URLs, got %d", len(urls))
66+
}
67+
68+
expectedURLs := []string{"http://example.com/path1", "https://example.com/path2"}
69+
if len(urls) != len(expectedURLs) {
70+
t.Fatalf("Expected %d URLs, got %d: %v", len(expectedURLs), len(urls), urls)
71+
}
72+
for i, expected := range expectedURLs {
73+
if urls[i] != expected {
74+
t.Errorf("Expected URL %d to be '%s', got '%s'", i, expected, urls[i])
75+
}
76+
}
77+
}
78+
79+
func TestBurpFormat_ParseEmpty(t *testing.T) {
80+
burpXML := `<?xml version="1.0"?>
81+
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
82+
</items>`
83+
84+
b := NewBurpFormat()
85+
var urls []string
86+
87+
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
88+
urls = append(urls, url)
89+
return true
90+
})
91+
92+
if err != nil {
93+
t.Fatalf("Parse returned error: %v", err)
94+
}
95+
96+
if len(urls) != 0 {
97+
t.Errorf("Expected 0 URLs, got %d", len(urls))
98+
}
99+
}
100+
101+
func TestBurpFormat_ParseStopEarly(t *testing.T) {
102+
burpXML := `<?xml version="1.0"?>
103+
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
104+
<item>
105+
<url><![CDATA[http://example.com/1]]></url>
106+
<host>example.com</host>
107+
<port>80</port>
108+
<protocol>http</protocol>
109+
<method><![CDATA[GET]]></method>
110+
<path><![CDATA[/1]]></path>
111+
<extension>null</extension>
112+
<request base64="true"><![CDATA[R0VUIC8=]]></request>
113+
<status>200</status>
114+
<responselength>100</responselength>
115+
<mimetype>HTML</mimetype>
116+
<response base64="true"><![CDATA[T0s=]]></response>
117+
<comment></comment>
118+
</item>
119+
<item>
120+
<url><![CDATA[http://example.com/2]]></url>
121+
<host>example.com</host>
122+
<port>80</port>
123+
<protocol>http</protocol>
124+
<method><![CDATA[GET]]></method>
125+
<path><![CDATA[/2]]></path>
126+
<extension>null</extension>
127+
<request base64="true"><![CDATA[R0VUIC8=]]></request>
128+
<status>200</status>
129+
<responselength>100</responselength>
130+
<mimetype>HTML</mimetype>
131+
<response base64="true"><![CDATA[T0s=]]></response>
132+
<comment></comment>
133+
</item>
134+
</items>`
135+
136+
b := NewBurpFormat()
137+
var urls []string
138+
139+
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
140+
urls = append(urls, url)
141+
return false // stop after first
142+
})
143+
144+
if err != nil {
145+
t.Fatalf("Parse returned error: %v", err)
146+
}
147+
148+
if len(urls) != 1 {
149+
t.Errorf("Expected 1 URL (stopped early), got %d", len(urls))
150+
}
151+
}
152+
153+
func TestBurpFormat_ParseMalformed(t *testing.T) {
154+
malformedXML := `<?xml version="1.0"?>
155+
<items burpVersion="2023.10.1.2">
156+
<item>
157+
<url><![CDATA[http://example.com/path1]]></url>
158+
<!-- missing closing tags -->
159+
</items>`
160+
161+
b := NewBurpFormat()
162+
err := b.Parse(strings.NewReader(malformedXML), func(url string) bool {
163+
return true
164+
})
165+
166+
if err == nil {
167+
t.Error("Expected error for malformed XML, got nil")
168+
}
169+
}

‎common/inputformats/formats.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// TODO: This package should be abstracted out to projectdiscovery/utils
2+
// so it can be shared between httpx, nuclei, and other tools.
3+
package inputformats
4+
5+
import (
6+
"io"
7+
"strings"
8+
)
9+
10+
// Format is an interface implemented by all input formats
11+
type Format interface {
12+
// Name returns the name of the format
13+
Name() string
14+
// Parse parses the input and calls the provided callback
15+
// function for each URL it discovers.
16+
Parse(input io.Reader, callback func(url string) bool) error
17+
}
18+
19+
// Supported formats
20+
var formats = []Format{
21+
NewBurpFormat(),
22+
}
23+
24+
// GetFormat returns the format by name
25+
func GetFormat(name string) Format {
26+
for _, f := range formats {
27+
if strings.EqualFold(f.Name(), name) {
28+
return f
29+
}
30+
}
31+
return nil
32+
}
33+
34+
// SupportedFormats returns a comma-separated list of supported format names
35+
func SupportedFormats() string {
36+
var names []string
37+
for _, f := range formats {
38+
names = append(names, f.Name())
39+
}
40+
return strings.Join(names, ", ")
41+
}

‎common/inputformats/formats_test.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package inputformats
2+
3+
import (
4+
"strings"
5+
"testing"
6+
)
7+
8+
func TestGetFormat(t *testing.T) {
9+
tests := []struct {
10+
name string
11+
input string
12+
wantNil bool
13+
wantName string
14+
}{
15+
{"burp lowercase", "burp", false, "burp"},
16+
{"burp uppercase", "BURP", false, "burp"},
17+
{"burp mixed case", "Burp", false, "burp"},
18+
{"invalid format", "invalid", true, ""},
19+
{"empty string", "", true, ""},
20+
}
21+
22+
for _, tt := range tests {
23+
t.Run(tt.name, func(t *testing.T) {
24+
got := GetFormat(tt.input)
25+
if tt.wantNil && got != nil {
26+
t.Errorf("GetFormat(%q) = %v, want nil", tt.input, got)
27+
}
28+
if !tt.wantNil && got == nil {
29+
t.Errorf("GetFormat(%q) = nil, want non-nil", tt.input)
30+
}
31+
if !tt.wantNil && got != nil && got.Name() != tt.wantName {
32+
t.Errorf("GetFormat(%q).Name() = %q, want %q", tt.input, got.Name(), tt.wantName)
33+
}
34+
})
35+
}
36+
}
37+
38+
func TestSupportedFormats(t *testing.T) {
39+
supported := SupportedFormats()
40+
if !strings.Contains(supported, "burp") {
41+
t.Errorf("SupportedFormats() = %q, expected to contain 'burp'", supported)
42+
}
43+
}

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ require (
5454
github.com/dustin/go-humanize v1.0.1
5555
github.com/go-viper/mapstructure/v2 v2.4.0
5656
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
57+
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193
58+
github.com/seh-msft/burpxml v1.0.1
5759
github.com/weppos/publicsuffix-go v0.50.2
5860
)
5961

@@ -128,7 +130,6 @@ require (
128130
github.com/pierrec/lz4/v4 v4.1.23 // indirect
129131
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
130132
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
131-
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect
132133
github.com/projectdiscovery/blackrock v0.0.1 // indirect
133134
github.com/projectdiscovery/freeport v0.0.7 // indirect
134135
github.com/projectdiscovery/gostruct v0.0.2 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,8 @@ github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uR
389389
github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
390390
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
391391
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
392+
github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg=
393+
github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I=
392394
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
393395
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
394396
github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=

‎runner/options.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
customport "github.com/projectdiscovery/httpx/common/customports"
2424
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
2525
httpxcommon "github.com/projectdiscovery/httpx/common/httpx"
26+
"github.com/projectdiscovery/httpx/common/inputformats"
2627
"github.com/projectdiscovery/httpx/common/stringz"
2728
"github.com/projectdiscovery/networkpolicy"
2829
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -191,6 +192,7 @@ type Options struct {
191192
SocksProxy string
192193
Proxy string
193194
InputFile string
195+
InputMode string
194196
InputTargetHost goflags.StringSlice
195197
Methods string
196198
RequestURI string
@@ -377,6 +379,7 @@ func ParseOptions() *Options {
377379
flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
378380
flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
379381
flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions),
382+
flagSet.StringVarP(&options.InputMode, "input-mode", "im", "", fmt.Sprintf("mode of input file (%s)", inputformats.SupportedFormats())),
380383
)
381384

382385
flagSet.CreateGroup("Probes", "Probes",
@@ -683,6 +686,13 @@ func (options *Options) ValidateOptions() error {
683686
if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) {
684687
return fmt.Errorf("secret file '%s' does not exist", options.SecretFile)
685688
}
689+
if options.InputMode != "" && inputformats.GetFormat(options.InputMode) == nil {
690+
return fmt.Errorf("invalid input mode '%s', supported formats: %s", options.InputMode, inputformats.SupportedFormats())
691+
}
692+
693+
if options.InputMode != "" && options.InputFile == "" {
694+
return errors.New("-im/-input-mode requires -l/-list to specify an input file")
695+
}
686696

687697
if options.Silent {
688698
incompatibleFlagsList := flagsIncompatibleWithSilent(options)

0 commit comments

Comments
 (0)