Merge pull request #85 from R3dTr4p/feature/bulk-download-reports

Feature/bulk download reports

Author: sw33tLie

cmd/reports.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var reportsCmd = &cobra.Command{
+	Use:   "reports",
+	Short: "Download bug bounty reports as Markdown files",
+}
+
+func init() {
+	rootCmd.AddCommand(reportsCmd)
+
+	reportsCmd.PersistentFlags().String("output-dir", "reports", "Output directory for downloaded reports")
+	reportsCmd.PersistentFlags().StringSlice("program", nil, "Filter by program handle(s)")
+	reportsCmd.PersistentFlags().StringSlice("state", nil, "Filter by report state(s) (e.g. resolved,triaged)")
+	reportsCmd.PersistentFlags().StringSlice("severity", nil, "Filter by severity (e.g. high,critical)")
+	reportsCmd.PersistentFlags().Bool("dry-run", false, "List reports without downloading")
+	reportsCmd.PersistentFlags().Bool("overwrite", false, "Overwrite existing report files")
+}

cmd/reports_h1.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"sync/atomic"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/sw33tLie/bbscope/v2/internal/utils"
+	"github.com/sw33tLie/bbscope/v2/pkg/reports"
+	"github.com/sw33tLie/bbscope/v2/pkg/whttp"
+)
+
+var reportsH1Cmd = &cobra.Command{
+	Use:   "h1",
+	Short: "Download reports from HackerOne",
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		user := viper.GetString("hackerone.username")
+		token := viper.GetString("hackerone.token")
+		if user == "" || token == "" {
+			utils.Log.Error("hackerone requires a username and token")
+			return nil
+		}
+
+		proxy, _ := rootCmd.Flags().GetString("proxy")
+		if proxy != "" {
+			whttp.SetupProxy(proxy)
+		}
+
+		outputDir, _ := cmd.Flags().GetString("output-dir")
+		programs, _ := cmd.Flags().GetStringSlice("program")
+		states, _ := cmd.Flags().GetStringSlice("state")
+		severities, _ := cmd.Flags().GetStringSlice("severity")
+		dryRun, _ := cmd.Flags().GetBool("dry-run")
+		overwrite, _ := cmd.Flags().GetBool("overwrite")
+
+		fetcher := reports.NewH1Fetcher(user, token)
+		opts := reports.FetchOptions{
+			Programs:   programs,
+			States:     states,
+			Severities: severities,
+			DryRun:     dryRun,
+			Overwrite:  overwrite,
+			OutputDir:  outputDir,
+		}
+
+		return runReportsH1(cmd.Context(), fetcher, opts)
+	},
+}
+
+func init() {
+	reportsCmd.AddCommand(reportsH1Cmd)
+	reportsH1Cmd.Flags().StringP("user", "u", "", "HackerOne username")
+	reportsH1Cmd.Flags().StringP("token", "t", "", "HackerOne API token")
+	viper.BindPFlag("hackerone.username", reportsH1Cmd.Flags().Lookup("user"))
+	viper.BindPFlag("hackerone.token", reportsH1Cmd.Flags().Lookup("token"))
+}
+
+func runReportsH1(ctx context.Context, fetcher *reports.H1Fetcher, opts reports.FetchOptions) error {
+	utils.Log.Info("Fetching report list from HackerOne...")
+
+	summaries, err := fetcher.ListReports(ctx, opts)
+	if err != nil {
+		return fmt.Errorf("listing reports: %w", err)
+	}
+
+	utils.Log.Infof("Found %d reports", len(summaries))
+
+	if len(summaries) == 0 {
+		return nil
+	}
+
+	// Dry-run: print table and exit
+	if opts.DryRun {
+		w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+		fmt.Fprintln(w, "ID\tPROGRAM\tSTATE\tSEVERITY\tCREATED\tTITLE")
+		for _, s := range summaries {
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
+				s.ID, s.ProgramHandle, s.Substate, s.SeverityRating, s.CreatedAt, s.Title)
+		}
+		w.Flush()
+		return nil
+	}
+
+	// Download mode with worker pool
+	var written, skipped, errored atomic.Int32
+	total := len(summaries)
+
+	workers := 10
+	if total < workers {
+		workers = total
+	}
+
+	jobs := make(chan int, total)
+	for i := range summaries {
+		jobs <- i
+	}
+	close(jobs)
+
+	var wg sync.WaitGroup
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range jobs {
+				s := summaries[i]
+				utils.Log.Infof("[%d/%d] Fetching report %s: %s", i+1, total, s.ID, s.Title)
+
+				report, err := fetcher.FetchReport(ctx, s.ID)
+				if err != nil {
+					utils.Log.Warnf("Error fetching report %s: %v", s.ID, err)
+					errored.Add(1)
+					continue
+				}
+
+				ok, err := reports.WriteReport(report, opts.OutputDir, opts.Overwrite)
+				if err != nil {
+					utils.Log.Warnf("Error writing report %s: %v", s.ID, err)
+					errored.Add(1)
+					continue
+				}
+
+				if ok {
+					written.Add(1)
+				} else {
+					skipped.Add(1)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+
+	utils.Log.Infof("Done: %d written, %d skipped, %d errors", written.Load(), skipped.Load(), errored.Load())
+	return nil
+}

pkg/reports/hackerone.go

@@ -0,0 +1,210 @@
+package reports
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/sw33tLie/bbscope/v2/internal/utils"
+	"github.com/sw33tLie/bbscope/v2/pkg/whttp"
+	"github.com/tidwall/gjson"
+)
+
+// H1Fetcher fetches reports from the HackerOne Hacker API.
+type H1Fetcher struct {
+	authB64 string
+}
+
+// NewH1Fetcher creates a fetcher using the same base64 auth pattern as the poller.
+func NewH1Fetcher(username, token string) *H1Fetcher {
+	raw := username + ":" + token
+	return &H1Fetcher{authB64: base64.StdEncoding.EncodeToString([]byte(raw))}
+}
+
+// ListReports fetches all report summaries matching the given options.
+func (f *H1Fetcher) ListReports(ctx context.Context, opts FetchOptions) ([]ReportSummary, error) {
+	var summaries []ReportSummary
+
+	queryFilter := buildQueryFilter(opts)
+	currentURL := "https://api.hackerone.com/v1/hackers/me/reports?page%5Bsize%5D=100"
+	if queryFilter != "" {
+		currentURL += "&filter%5Bkeyword%5D=" + queryFilter
+	}
+
+	for {
+		body, err := f.doRequest(currentURL)
+		if err != nil {
+			return summaries, err
+		}
+		if body == "" {
+			break // non-retryable status, stop
+		}
+
+		count := int(gjson.Get(body, "data.#").Int())
+		for i := 0; i < count; i++ {
+			prefix := "data." + strconv.Itoa(i)
+			summary := ReportSummary{
+				ID:             gjson.Get(body, prefix+".id").String(),
+				Title:          gjson.Get(body, prefix+".attributes.title").String(),
+				State:          gjson.Get(body, prefix+".attributes.state").String(),
+				Substate:       gjson.Get(body, prefix+".attributes.substate").String(),
+				CreatedAt:      gjson.Get(body, prefix+".attributes.created_at").String(),
+				SeverityRating: gjson.Get(body, prefix+".relationships.severity.data.attributes.rating").String(),
+			}
+
+			// Program handle from relationships
+			summary.ProgramHandle = gjson.Get(body, prefix+".relationships.program.data.attributes.handle").String()
+
+			summaries = append(summaries, summary)
+		}
+
+		nextURL := gjson.Get(body, "links.next").String()
+		if nextURL == "" {
+			break
+		}
+		currentURL = nextURL
+	}
+
+	return summaries, nil
+}
+
+// FetchReport fetches the full detail of a single report by ID.
+func (f *H1Fetcher) FetchReport(ctx context.Context, reportID string) (*Report, error) {
+	url := "https://api.hackerone.com/v1/hackers/reports/" + reportID
+
+	body, err := f.doRequest(url)
+	if err != nil {
+		return nil, err
+	}
+	if body == "" {
+		return nil, fmt.Errorf("report %s: not found or not accessible", reportID)
+	}
+
+	r := &Report{
+		ID:                       gjson.Get(body, "data.id").String(),
+		Title:                    gjson.Get(body, "data.attributes.title").String(),
+		State:                    gjson.Get(body, "data.attributes.state").String(),
+		Substate:                 gjson.Get(body, "data.attributes.substate").String(),
+		CreatedAt:                formatTimestamp(gjson.Get(body, "data.attributes.created_at").String()),
+		TriagedAt:                formatTimestamp(gjson.Get(body, "data.attributes.triaged_at").String()),
+		ClosedAt:                 formatTimestamp(gjson.Get(body, "data.attributes.closed_at").String()),
+		DisclosedAt:              formatTimestamp(gjson.Get(body, "data.attributes.disclosed_at").String()),
+		VulnerabilityInformation: gjson.Get(body, "data.attributes.vulnerability_information").String(),
+		Impact:                   gjson.Get(body, "data.attributes.impact").String(),
+		ProgramHandle:            gjson.Get(body, "data.relationships.program.data.attributes.handle").String(),
+		SeverityRating:           gjson.Get(body, "data.relationships.severity.data.attributes.rating").String(),
+		CVSSScore:                gjson.Get(body, "data.relationships.severity.data.attributes.score").String(),
+		WeaknessName:             gjson.Get(body, "data.relationships.weakness.data.attributes.name").String(),
+		WeaknessCWE:              gjson.Get(body, "data.relationships.weakness.data.attributes.external_id").String(),
+		AssetIdentifier:          gjson.Get(body, "data.relationships.structured_scope.data.attributes.asset_identifier").String(),
+	}
+
+	// Bounty amounts — sum all bounty relationships
+	var totalBounty float64
+	bounties := gjson.Get(body, "data.relationships.bounties.data")
+	if bounties.Exists() {
+		bounties.ForEach(func(_, v gjson.Result) bool {
+			totalBounty += v.Get("attributes.amount").Float()
+			return true
+		})
+	}
+	if totalBounty > 0 {
+		r.BountyAmount = fmt.Sprintf("%.2f", totalBounty)
+	}
+
+	// CVE IDs
+	cves := gjson.Get(body, "data.attributes.cve_ids")
+	if cves.Exists() {
+		cves.ForEach(func(_, v gjson.Result) bool {
+			if id := v.String(); id != "" {
+				r.CVEIDs = append(r.CVEIDs, id)
+			}
+			return true
+		})
+	}
+
+	return r, nil
+}
+
+// doRequest performs an authenticated GET with retries and rate-limit handling.
+func (f *H1Fetcher) doRequest(url string) (string, error) {
+	retries := 3
+	for retries > 0 {
+		res, err := whttp.SendHTTPRequest(&whttp.WHTTPReq{
+			Method:  "GET",
+			URL:     url,
+			Headers: []whttp.WHTTPHeader{{Name: "Authorization", Value: "Basic " + f.authB64}},
+		}, nil)
+
+		if err != nil {
+			retries--
+			utils.Log.Warnf("HTTP request failed (%s), retrying: %v", url, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+
+		// Rate limited
+		if res.StatusCode == 429 {
+			utils.Log.Warn("Rate limited by HackerOne, waiting 60s...")
+			time.Sleep(60 * time.Second)
+			continue // don't decrement retries for rate limits
+		}
+
+		// Non-retryable errors
+		if res.StatusCode == 400 || res.StatusCode == 403 || res.StatusCode == 404 {
+			utils.Log.Warnf("Got status %d for %s, skipping", res.StatusCode, url)
+			return "", nil
+		}
+
+		if res.StatusCode != 200 {
+			retries--
+			utils.Log.Warnf("Got status %d for %s, retrying", res.StatusCode, url)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+
+		return res.BodyString, nil
+	}
+
+	return "", fmt.Errorf("failed to fetch %s after retries", url)
+}
+
+// buildQueryFilter builds a Lucene-syntax filter string for the H1 API.
+func buildQueryFilter(opts FetchOptions) string {
+	var parts []string
+
+	if len(opts.Programs) > 0 {
+		for _, p := range opts.Programs {
+			parts = append(parts, "team:"+p)
+		}
+	}
+
+	if len(opts.States) > 0 {
+		for _, s := range opts.States {
+			parts = append(parts, "substate:"+s)
+		}
+	}
+
+	if len(opts.Severities) > 0 {
+		for _, s := range opts.Severities {
+			parts = append(parts, "severity_rating:"+s)
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+// formatTimestamp converts an ISO 8601 timestamp to a human-readable format.
+func formatTimestamp(ts string) string {
+	if ts == "" {
+		return ""
+	}
+	t, err := time.Parse(time.RFC3339, ts)
+	if err != nil {
+		return ts // return as-is if unparseable
+	}
+	return t.UTC().Format("2006-01-02 15:04 UTC")
+}