Merge pull request #2365 from projectdiscovery/dev

httpx v1.8.0

Author: dogancanbakir

.github/workflows/build-test.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   lint:
     name: Lint Test
-    if: "${{ !endsWith(github.actor, '[bot]') }}"
+    if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

README.md

@@ -92,9 +92,10 @@ Usage:
 
 Flags:
 INPUT:
-   -l, -list string      input file containing list of hosts to process
-   -rr, -request string  file containing raw request
-   -u, -target string[]  input target host(s) to probe
+   -l, -list string              input file containing list of hosts to process
+   -rr, -request string          file containing raw request
+   -u, -target string[]          input target host(s) to probe
+   -im, -input-mode string       mode of input file (burp)
 
 PROBES:
    -sc, -status-code      display response status-code
@@ -110,9 +111,11 @@ PROBES:
    -title                 display page title
    -bp, -body-preview     display first N characters of response body (default 100)
    -server, -web-server   display server name
-   -td, -tech-detect      display technology in use based on wappalyzer dataset
+   -td, -tech-detect                      display technology in use based on wappalyzer dataset
    -cff, -custom-fingerprint-file string  path to a custom fingerprint file for technology detection
-   -method                display http request method
+   -cpe                                   display CPE (Common Platform Enumeration) based on awesome-search-queries
+   -wp, -wordpress                        display WordPress plugins and themes
+   -method                                display http request method
    -ws, -websocket        display server using websocket
    -ip                    display host ip
    -cname                 display host cname
@@ -231,6 +234,7 @@ CONFIGURATIONS:
    -tlsi, -tls-impersonate          enable experimental client hello (ja3) tls randomization
    -no-stdin                        Disable Stdin processing
    -hae, -http-api-endpoint string  experimental http api endpoint
+   -sf, -secret-file string         path to secret file for authentication
 
 DEBUG:
    -health-check, -hc        run diagnostic check up
@@ -277,9 +281,28 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
 # Notes
 
 - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
+- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
 - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
 - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
+- Secret files can be used for domain-based authentication via `-sf secrets.yaml`. Supported auth types: `BasicAuth`, `BearerToken`, `Header`, `Cookie`, `Query`. Example:
+  ```yaml
+  id: example-auth
+  info:
+    name: Example Auth Config
+  static:
+    - type: Header
+      domains:
+        - api.example.com
+      headers:
+        - key: X-API-Key
+          value: secret-key-here
+    - type: BasicAuth
+      domains-regex:
+        - ".*\\.internal\\.com$"
+      username: admin
+      password: secret
+  ```
 - The following flags should be used for specific use cases instead of running them as default with other probes:
    - `-ports`
    - `-path`
@@ -307,4 +330,4 @@ Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomno
 
 <a href="https://discord.gg/projectdiscovery"><img src="https://raw.githubusercontent.com/projectdiscovery/nuclei-burp-plugin/main/static/join-discord.png" width="300" alt="Join Discord"></a>
 
-</div>
+</div>

common/authprovider/authx/basic_auth.go

@@ -0,0 +1,31 @@
+package authx
+
+import (
+	"net/http"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+var (
+	_ AuthStrategy = &BasicAuthStrategy{}
+)
+
+// BasicAuthStrategy is a strategy for basic auth
+type BasicAuthStrategy struct {
+	Data *Secret
+}
+
+// NewBasicAuthStrategy creates a new basic auth strategy
+func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy {
+	return &BasicAuthStrategy{Data: data}
+}
+
+// Apply applies the basic auth strategy to the request
+func (s *BasicAuthStrategy) Apply(req *http.Request) {
+	req.SetBasicAuth(s.Data.Username, s.Data.Password)
+}
+
+// ApplyOnRR applies the basic auth strategy to the retryable request
+func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
+	req.SetBasicAuth(s.Data.Username, s.Data.Password)
+}

common/authprovider/authx/bearer_auth.go

@@ -0,0 +1,31 @@
+package authx
+
+import (
+	"net/http"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+var (
+	_ AuthStrategy = &BearerTokenAuthStrategy{}
+)
+
+// BearerTokenAuthStrategy is a strategy for bearer token auth
+type BearerTokenAuthStrategy struct {
+	Data *Secret
+}
+
+// NewBearerTokenAuthStrategy creates a new bearer token auth strategy
+func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy {
+	return &BearerTokenAuthStrategy{Data: data}
+}
+
+// Apply applies the bearer token auth strategy to the request
+func (s *BearerTokenAuthStrategy) Apply(req *http.Request) {
+	req.Header.Set("Authorization", "Bearer "+s.Data.Token)
+}
+
+// ApplyOnRR applies the bearer token auth strategy to the retryable request
+func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
+	req.Header.Set("Authorization", "Bearer "+s.Data.Token)
+}

common/authprovider/authx/cookies_auth.go

@@ -0,0 +1,62 @@
+package authx
+
+import (
+	"net/http"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+var (
+	_ AuthStrategy = &CookiesAuthStrategy{}
+)
+
+// CookiesAuthStrategy is a strategy for cookies auth
+type CookiesAuthStrategy struct {
+	Data *Secret
+}
+
+// NewCookiesAuthStrategy creates a new cookies auth strategy
+func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy {
+	return &CookiesAuthStrategy{Data: data}
+}
+
+// Apply applies the cookies auth strategy to the request
+func (s *CookiesAuthStrategy) Apply(req *http.Request) {
+	for _, cookie := range s.Data.Cookies {
+		req.AddCookie(&http.Cookie{
+			Name:  cookie.Key,
+			Value: cookie.Value,
+		})
+	}
+}
+
+// ApplyOnRR applies the cookies auth strategy to the retryable request
+func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
+	// Build a set of cookie names to replace
+	newCookieNames := make(map[string]struct{}, len(s.Data.Cookies))
+	for _, cookie := range s.Data.Cookies {
+		newCookieNames[cookie.Key] = struct{}{}
+	}
+
+	// Filter existing cookies, keeping only those not being replaced
+	existingCookies := req.Cookies()
+	filteredCookies := make([]*http.Cookie, 0, len(existingCookies))
+	for _, cookie := range existingCookies {
+		if _, shouldReplace := newCookieNames[cookie.Name]; !shouldReplace {
+			filteredCookies = append(filteredCookies, cookie)
+		}
+	}
+
+	// Clear and reset cookies
+	req.Header.Del("Cookie")
+	for _, cookie := range filteredCookies {
+		req.AddCookie(cookie)
+	}
+	// Add new cookies
+	for _, cookie := range s.Data.Cookies {
+		req.AddCookie(&http.Cookie{
+			Name:  cookie.Key,
+			Value: cookie.Value,
+		})
+	}
+}