Merge pull request #2435 from projectdiscovery/dev

release httpx v1.9.0

Author: dogancanbakir

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+## Proposed changes
+
+<!-- Describe the overall picture of your modifications to help maintainers understand the pull request. PRs are required to be associated to their related issue tickets or feature request. -->
+
+### Proof
+
+<!-- How has this been tested? Please describe the tests that you ran to verify your changes. -->
+
+## Checklist
+
+<!-- Put an "x" in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code. -->
+
+- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/httpx/tree/dev) branch
+- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] I have added necessary documentation (if appropriate)

.github/workflows/build-test.yml

@@ -15,12 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: projectdiscovery/actions/setup/go@v1
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v5
-        with:
-          version: latest
-          args: --timeout 5m
-          working-directory: .
+      - uses: projectdiscovery/actions/golangci-lint/v2@v1
 
   build:
     name: Test Builds

.github/workflows/functional-test.yml

@@ -7,21 +7,18 @@ on:
       - '**.mod'
   workflow_dispatch:
 
-jobs:  
+jobs:
   functional:
     name: Functional Test
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - uses: projectdiscovery/actions/setup/go@v1
 
       - name: Functional Tests
         run: |

.github/workflows/release-binary.yml

@@ -11,23 +11,20 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v3
-        with: 
+        uses: actions/checkout@v4
+        with:
           fetch-depth: 0
-      
-      - name: "Set up Go"
-        uses: actions/setup-go@v4
-        with: 
-          go-version: 1.21.x
-      
+
+      - uses: projectdiscovery/actions/setup/go@v1
+
       - name: "Create release on GitHub"
         uses: goreleaser/goreleaser-action@v4
-        with: 
+        with:
           args: "release --clean"
           version: latest
           workdir: .
-        env: 
+        env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
           DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}"
-          DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}"
+          DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}"

.github/workflows/release-test.yml

@@ -12,15 +12,12 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v3
-        with: 
+        uses: actions/checkout@v4
+        with:
           fetch-depth: 0
 
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-      
+      - uses: projectdiscovery/actions/setup/go@v1
+
       - name: release test
         uses: goreleaser/goreleaser-action@v4
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # Base
-FROM golang:1.24.5-alpine AS builder
+FROM golang:1.25.7-alpine AS builder
 
 RUN apk add --no-cache git build-base gcc musl-dev
 WORKDIR /app

README.md

@@ -62,7 +62,7 @@
 
 # Installation Instructions
 
-`httpx` requires **go >=1.24.0** to install successfully. Run the following command to get the repo:
+`httpx` requires **go >=1.25.0** to install successfully. Run the following command to get the repo:
 
 ```sh
 go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
@@ -98,42 +98,40 @@ INPUT:
    -im, -input-mode string       mode of input file (burp)
 
 PROBES:
-   -sc, -status-code      display response status-code
-   -cl, -content-length   display response content-length
-   -ct, -content-type     display response content-type
-   -location              display response redirect location
-   -favicon               display mmh3 hash for '/favicon.ico' file
-   -hash string           display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
-   -jarm                  display jarm fingerprint hash
-   -rt, -response-time    display response time
-   -lc, -line-count       display response body line count
-   -wc, -word-count       display response body word count
-   -title                 display page title
-   -bp, -body-preview     display first N characters of response body (default 100)
-   -server, -web-server   display server name
+   -sc, -status-code                      display response status-code
+   -cl, -content-length                   display response content-length
+   -ct, -content-type                     display response content-type
+   -location                              display response redirect location
+   -favicon                               display mmh3 hash for '/favicon.ico' file
+   -hash string                           display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
+   -jarm                                  display jarm fingerprint hash
+   -rt, -response-time                    display response time
+   -lc, -line-count                       display response body line count
+   -wc, -word-count                       display response body word count
+   -title                                 display page title
+   -bp, -body-preview                     display first N characters of response body (default 100)
+   -server, -web-server                   display server name
    -td, -tech-detect                      display technology in use based on wappalyzer dataset
    -cff, -custom-fingerprint-file string  path to a custom fingerprint file for technology detection
-   -cpe                                   display CPE (Common Platform Enumeration) based on awesome-search-queries
-   -wp, -wordpress                        display WordPress plugins and themes
    -method                                display http request method
-   -ws, -websocket        display server using websocket
-   -ip                    display host ip
-   -cname                 display host cname
-   -extract-fqdn, -efqdn  get domain and subdomains from response body and header in jsonl/csv output
-   -asn                   display host asn information
-   -cdn                   display cdn/waf in use (default true)
-   -probe                 display probe status
+   -ws, -websocket                        display server using websocket
+   -ip                                    display host ip
+   -cname                                 display host cname
+   -extract-fqdn, -efqdn                  get domain and subdomains from response body and header in jsonl/csv output
+   -asn                                   display host asn information
+   -cdn                                   display cdn/waf in use (default true)
+   -probe                                 display probe status
 
 HEADLESS:
    -ss, -screenshot                 enable saving screenshot of the page using headless browser
    -system-chrome                   enable using local installed chrome for screenshot
    -ho, -headless-options string[]  start headless chrome with additional options
    -esb, -exclude-screenshot-bytes  enable excluding screenshot bytes from json output
-   -no-screenshot-full-page         disable saving full page screenshot
    -ehb, -exclude-headless-body     enable excluding headless header from json output
+   -no-screenshot-full-page         disable saving full page screenshot
    -st, -screenshot-timeout value   set timeout for screenshot in seconds (default 10s)
    -sid, -screenshot-idle value     set idle time before taking screenshot in seconds (default 1s)
-   -jsc, -javascript-code string[]   execute JavaScript code after navigation
+   -jsc, -javascript-code string[]  execute JavaScript code after navigation
 
 MATCHERS:
    -mc, -match-code string            match response with specified status code (-mc 200,302)
@@ -143,7 +141,7 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string[]        match response with specified string (-ms admin)
    -mr, -match-regex string[]         match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (cloudfront, fastly, google)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (cloudfront, fastly, google, etc.)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
@@ -152,19 +150,22 @@ EXTRACTOR:
    -ep, -extract-preset string[]  display response content matched by a pre-defined regex (url,ipv4,mail)
 
 FILTERS:
-   -fc, -filter-code string            filter response with specified status code (-fc 403,401)
-   -fep, -filter-error-page            filter response with ML based error page detection
-   -fd, -filter-duplicates             filter out near-duplicate responses (only first response is retained)
-   -fl, -filter-length string          filter response with specified content length (-fl 23,33)
-   -flc, -filter-line-count string     filter response body with specified line count (-flc 423,532)
-   -fwc, -filter-word-count string     filter response body with specified word count (-fwc 423,532)
-   -ffc, -filter-favicon string[]      filter response with specified favicon hash (-ffc 1494302000)
-   -fs, -filter-string string[]        filter response with specified string (-fs admin)
-   -fe, -filter-regex string[]         filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (cloudfront, fastly, google)
-   -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
-   -fdc, -filter-condition string      filter response with dsl expression condition
-   -strip                              strips all tags in response. supported formats: html,xml (default html)
+   -fc, -filter-code string               filter response with specified status code (-fc 403,401)
+   -fpt, -filter-page-type string[]       filter response with specified page type (e.g. -fpt login,captcha,parked)
+   -fep, -filter-error-page               [DEPRECATED: use -fpt] filter response with ML based error page detection
+   -fd, -filter-duplicates                filter out near-duplicate responses (only first response is retained)
+   -fl, -filter-length string             filter response with specified content length (-fl 23,33)
+   -flc, -filter-line-count string        filter response body with specified line count (-flc 423,532)
+   -fwc, -filter-word-count string        filter response body with specified word count (-fwc 423,532)
+   -ffc, -filter-favicon string[]         filter response with specified favicon hash (-ffc 1494302000)
+   -fs, -filter-string string[]           filter response with specified string (-fs admin)
+   -fe, -filter-regex string[]            filter response with specified regex (-fe admin)
+   -fcdn, -filter-cdn string[]            filter host with specified cdn provider (cloudfront, fastly, google, etc.)
+   -frt, -filter-response-time string     filter response with specified response time in seconds (-frt '> 1')
+   -fdc, -filter-condition string         filter response with dsl expression condition
+   -strip                                 strips all tags in response. supported formats: html,xml (default html)
+   -lof, -list-output-fields              list of fields to output (comma separated)
+   -eof, -exclude-output-fields string[]  exclude output fields output based on a condition
 
 RATE-LIMIT:
    -t, -threads int              number of threads to use (default 50)
@@ -202,10 +203,16 @@ OUTPUT:
    -include-chain                         include redirect http chain in JSON output (-json only)
    -store-chain                           include http redirect chain in responses (-sr only)
    -svrc, -store-vision-recon-cluster     include visual recon clusters (-ss and -sr only)
-   -pr, -protocol string                  protocol to use (unknown, http11)
+   -pr, -protocol string                  protocol to use (unknown, http11, http2, http3)
    -fepp, -filter-error-page-path string  path to store filtered error pages (default "filtered_error_page.json")
-   -lof, -list-output-fields              list available output field names for filtering
-   -eof, -exclude-output-fields string[]  exclude specified output fields from results
+   -rdb, -result-db                       store results in database
+   -rdbc, -result-db-config string        path to database config file
+   -rdbt, -result-db-type string          database type (mongodb, postgres, mysql)
+   -rdbcs, -result-db-conn string         database connection string (env: HTTPX_DB_CONNECTION_STRING)
+   -rdbn, -result-db-name string          database name (default "httpx")
+   -rdbtb, -result-db-table string        table/collection name (default "results")
+   -rdbbs, -result-db-batch-size int      batch size for database inserts (default 100)
+   -rdbor, -result-db-omit-raw            omit raw request/response data from database
 
 CONFIGURATIONS:
    -config string                   path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
@@ -214,9 +221,9 @@ CONFIGURATIONS:
    -deny string[]                   denied list of IP/CIDR's to process (file or comma separated)
    -sni, -sni-name string           custom TLS SNI name
    -random-agent                    enable Random User-Agent to use (default true)
-   -auto-referer                    set the Referer header to the current URL (default false)
+   -auto-referer                    set the Referer header to the current URL
    -H, -header string[]             custom http headers to send with request
-   -http-proxy, -proxy string       http proxy to use (eg http://127.0.0.1:8080)
+   -http-proxy, -proxy string       proxy (http|socks) to use (eg http://127.0.0.1:8080)
    -unsafe                          send raw requests skipping golang normalization
    -resume                          resume scan using resume.cfg
    -fr, -follow-redirects           follow http redirects
@@ -252,14 +259,14 @@ DEBUG:
 
 OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
-   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input 
+   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input
    -maxhr, -max-host-error int        max error count per host before skipping remaining path/s (default 30)
    -e, -exclude string[]              exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
    -retries int                       number of retries
    -timeout int                       timeout in seconds (default 10)
    -delay value                       duration between each http request (eg: 200ms, 1s) (default -1ns)
-   -rsts, -response-size-to-save int  max response size to save in bytes (default 2147483647)
-   -rstr, -response-size-to-read int  max response size to read in bytes (default 2147483647)
+   -rsts, -response-size-to-save int  max response size to save in bytes (default 512000000)
+   -rstr, -response-size-to-read int  max response size to read in bytes (default 512000000)
 
 CLOUD:
    -auth                           configure projectdiscovery cloud (pdcp) api key (default true)