diff --git a/core/event.go b/core/event.go new file mode 100644 index 0000000..800547e --- /dev/null +++ b/core/event.go @@ -0,0 +1,48 @@ +package core + +import ( + "encoding/json" + "fmt" + "strings" +) + +// A MatchEvent holds a list of matches for a particular Url and signature +type MatchEvent struct { + Url string + Matches []string + Signature string + File string + Stars int + Source GitResourceType +} + +// Line returns a string containing the following: Url, Signature, File, and +// Matches +func (m MatchEvent) String() string { + var b strings.Builder + + b.WriteString(fmt.Sprintf("Url: %s ", m.Url)) + b.WriteString(fmt.Sprintf("Signature: %s ", m.Signature)) + b.WriteString(fmt.Sprintf("File: %s ", m.File)) + b.WriteString(fmt.Sprintf("Matches: %s\n", strings.Join(m.Matches, ", "))) + + return b.String() +} + +// Line returns a slice of strings containing the following: Url, Signature, +// File, and Matches +func (m MatchEvent) Line() []string { + return []string{m.Url, m.Signature, m.File, strings.Join(m.Matches, ", ")} +} + +// Json returns a JSON formatted string that includes all of the data in a +// MatchEvent. +func (m MatchEvent) Json() string { + b, err := json.Marshal(m) + if err != nil { + LogIfError("unable to create JSON, %s", err) + return "" + } + + return string(b) +} diff --git a/core/options.go b/core/options.go index 52d0119..5522e47 100644 --- a/core/options.go +++ b/core/options.go @@ -19,6 +19,7 @@ type Options struct { ProcessGists *bool TempDirectory *string CsvPath *string + Delimiter *string SearchQuery *string Local *string Live *string @@ -39,6 +40,7 @@ func ParseOptions() (*Options, error) { ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."), TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"), CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"), + Delimiter: flag.String("delimiter", ",", "Delimiter for CSV file."), SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"), Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have Githib tokens with local run."), Live: flag.String("live", "", "Your shhgit live endpoint"), diff --git a/core/publish.go b/core/publish.go new file mode 100644 index 0000000..63b3257 --- /dev/null +++ b/core/publish.go @@ -0,0 +1,105 @@ +package core + +import ( + "bytes" + "encoding/csv" + "fmt" + "net/http" + "os" + "time" +) + +type Publisher interface { + Publish(m MatchEvent) error +} + +// WebSource represents a web-based publisher +type WebSource struct { + client *http.Client + endpoint string + method string + contentType string +} + +// Publish writes a MatchEvent to the WebSource. +func (w *WebSource) Publish(m MatchEvent) error { + var data string + + switch w.contentType { + case "application/json": + data = m.Json() + // Add new cases for additional content types + default: + // Nothing + } + + req, err := http.NewRequest(w.method, w.endpoint, bytes.NewBufferString(data)) + if err != nil { + return err + } + + req.Header.Add("Content-Type", w.contentType) + _, err = w.client.Do(req) + if err != nil { + return err + } + + // May need to capture the response and check for status codes like 404 or + // 503 so that errors can be returned if needed. + + return nil +} + +// Returns a new WebSource that will send MatchEvents to the given enpoint +// using the given method and content type. +func NewWebSource(endpoint, method, contentType string) (WebSource, error) { + var w WebSource + + if !(method == "POST" || method == "GET") { + return w, fmt.Errorf("method must be POST or GET, received %s", method) + } + + w.client = &http.Client{Timeout: 10 * time.Second} + w.endpoint = endpoint + w.method = method + w.contentType = contentType + + return w, nil +} + +// DelimitedSource represents a Comma or Tab delimited publisher. +type DelimitedSource struct { + writer *csv.Writer +} + +// Publish writes a MatchEvent to the file. +func (w *DelimitedSource) Publish(m MatchEvent) error { + err := w.writer.Write(m.Line()) + if err != nil { + return err + } + + w.writer.Flush() + + return nil +} + +// NewDelimitedSource returns a new comma or tab delimited writer. If a header +// is provided, it is written to the file. +func NewDelimitedSource(filename string, delimiter rune, header []string) (DelimitedSource, error) { + var d DelimitedSource + + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return d, err + } + + d.writer = csv.NewWriter(file) + d.writer.Comma = delimiter + + if header != nil { + d.writer.Write(header) + } + + return d, nil +} diff --git a/core/session.go b/core/session.go index 261a9af..75eda72 100644 --- a/core/session.go +++ b/core/session.go @@ -2,7 +2,6 @@ package core import ( "context" - "encoding/csv" "fmt" "math/rand" "os" @@ -28,7 +27,7 @@ type Session struct { Context context.Context Clients chan *GitHubClientWrapper ExhaustedClients chan *GitHubClientWrapper - CsvWriter *csv.Writer + Publishers []Publisher } var ( @@ -44,7 +43,7 @@ func (s *Session) Start() { s.InitThreads() s.InitSignatures() s.InitGitHubClients() - s.InitCsvWriter() + s.InitPublishers() } func (s *Session) InitLogger() { @@ -130,33 +129,32 @@ func (s *Session) InitThreads() { runtime.GOMAXPROCS(*s.Options.Threads + 1) } -func (s *Session) InitCsvWriter() { - if *s.Options.CsvPath == "" { - return - } - - writeHeader := false - if !PathExists(*s.Options.CsvPath) { - writeHeader = true - } +func (s *Session) InitPublishers() { + header := []string{"Repository name", "Signature name", "Matching file", "Matches"} - file, err := os.OpenFile(*s.Options.CsvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - LogIfError("Could not create/open CSV file", err) - - s.CsvWriter = csv.NewWriter(file) - - if writeHeader { - s.WriteToCsv([]string{"Repository name", "Signature name", "Matching file", "Matches"}) + // Setup our delimited publisher + if *s.Options.CsvPath != "" { + // Convert delimiter to a rune slice. We only need the first rune + delimiter := []rune(*s.Options.Delimiter) + publisher, err := NewDelimitedSource(*s.Options.CsvPath, delimiter[0], header) + if err != nil { + s.Log.Error("Cannot create CSV publisher: %s", err) + } + if err == nil { + s.Publishers = append(s.Publishers, &publisher) + } } -} -func (s *Session) WriteToCsv(line []string) { - if *s.Options.CsvPath == "" { - return + // Setup our Live publisher + if *session.Options.Live != "" { + publisher, err := NewWebSource(*session.Options.Live, "POST", "application/json") + if err != nil { + s.Log.Error("Cannot create Live publisher: %s", err) + } + if err == nil { + s.Publishers = append(s.Publishers, &publisher) + } } - - s.CsvWriter.Write(line) - s.CsvWriter.Flush() } func GetSession() *Session { diff --git a/main.go b/main.go index 72cfe0d..53a4791 100644 --- a/main.go +++ b/main.go @@ -3,9 +3,7 @@ package main import ( "bufio" "bytes" - "encoding/json" "io/ioutil" - "net/http" "os" "path/filepath" "regexp" @@ -15,15 +13,6 @@ import ( "github.com/fatih/color" ) -type MatchEvent struct { - Url string - Matches []string - Signature string - File string - Stars int - Source core.GitResourceType -} - var session = core.GetSession() func ProcessRepositories() { @@ -125,8 +114,8 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT if matches != nil { count := len(matches) m := strings.Join(matches, ", ") + publish(core.MatchEvent{Url: url, Signature: "Search Query", File: relativeFileName, Matches: matches}) session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m)) - session.WriteToCsv([]string{url, "Search Query", relativeFileName, m}) } } else { for _, signature := range session.Signatures { @@ -137,15 +126,13 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT if matches = signature.GetContentsMatches(file.Contents); matches != nil { count := len(matches) m := strings.Join(matches, ", ") - publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) + publish(core.MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m)) - session.WriteToCsv([]string{url, signature.Name(), relativeFileName, m}) } } else { if *session.Options.PathChecks { - publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) + publish(core.MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) session.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name())) - session.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""}) } if *session.Options.EntropyThreshold > 0 && file.CanCheckEntropy() { @@ -167,9 +154,8 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT } if !blacklistedMatch { - publish(&MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars}) + publish(core.MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars}) session.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(line)) - session.WriteToCsv([]string{url, "High entropy string", relativeFileName, line}) } } } @@ -187,11 +173,12 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT return } -func publish(event *MatchEvent) { - // todo: implement a modular plugin system to handle the various outputs (console, live, csv, webhooks, etc) - if len(*session.Options.Live) > 0 { - data, _ := json.Marshal(event) - http.Post(*session.Options.Live, "application/json", bytes.NewBuffer(data)) +func publish(event core.MatchEvent) { + for _, publisher := range session.Publishers { + err := publisher.Publish(event) + if err != nil { + session.Log.Error("Cannot publish: %s", err) + } } }