Add proof of concept vulnfeeds code. (#100)

This is something which can find relevant vulnerabilities from existing
feeds such as CVEs and output them in a format compatible with OSV.

Author: oliverchang

vulnfeeds/.gitignore

@@ -0,0 +1 @@
+pypi.json

vulnfeeds/README.md

@@ -0,0 +1,25 @@
+# Vuln feeds
+This searches a NVD CVE JSON feed for relevant packages and creates
+vulnerability entries in the same easy-to-consume format that OSV uses.
+
+## Package name matching
+CVE entries do not provide an easy mapping to the exact package name used in a
+package manager, so we dump and compare reference URLs for the package against
+reference URLs used in the CVE, in addition to some other heuristics to avoid
+false positives.
+
+## PyPI
+For PyPI, we find package reference URLs by doing a BigQuery query on
+the public PyPI dataset:
+
+```bash
+bq query --max_rows=10000000 --format=json --nouse_legacy_sql --flagfile=pypi.sql > pypi.json
+```
+
+However this includes packages that no longer exist or were deleted, so we check
+against the [pypi simple API](https://warehouse.pypa.io/api-reference/legacy.html)
+to make sure any matches actually exist.
+
+## Extracting affected versions and commits
+Where possible, we try to extract affected version ranges from descriptions and
+other fields, and extract commit hashes from e.g. GitHub links.

vulnfeeds/cmd/pypi/main.go

@@ -0,0 +1,75 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/google/osv/vulnfeeds/cves"
+	"github.com/google/osv/vulnfeeds/pypi"
+	"github.com/google/osv/vulnfeeds/vulns"
+)
+
+func main() {
+	jsonPath := flag.String("nvd_json", "", "Path to NVD CVE JSON.")
+	pypiJson := flag.String("pypi_json", "", "Path to pypi.json.")
+	outDir := flag.String("out_dir", "", "Path to output results.")
+
+	flag.Parse()
+
+	data, err := ioutil.ReadFile(*jsonPath)
+	if err != nil {
+		log.Fatalf("Failed to open file: %v", err)
+	}
+
+	var parsed cves.NVDCVE
+	err = json.Unmarshal(data, &parsed)
+	if err != nil {
+		log.Fatalf("Failed to parse NVD CVE JSON: %v", err)
+	}
+
+	ecosystem := pypi.NewPyPI(*pypiJson)
+	for _, cve := range parsed.CVEItems {
+		pkg := ""
+		if pkg = ecosystem.Matches(cve); pkg == "" {
+			continue
+		}
+
+		v := vulns.FromCVE(cve, pkg, "PyPI", "ECOSYSTEM")
+		data, err := yaml.Marshal(v)
+		if err != nil {
+			log.Fatalf("Failed to marshal YAML: %v", err)
+		}
+
+		pkgDir := filepath.Join(*outDir, pkg)
+		err = os.MkdirAll(pkgDir, 0755)
+		if err != nil {
+			log.Fatalf("Failed to create dir: %v", err)
+		}
+
+		vulnPath := filepath.Join(pkgDir, v.ID+".yaml")
+		err = ioutil.WriteFile(vulnPath, data, 0644)
+		if err != nil {
+			log.Fatalf("Failed to write %s: %v", vulnPath, err)
+		}
+	}
+}

vulnfeeds/cves/cve.go

@@ -0,0 +1,81 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cves
+
+import "time"
+
+const (
+	CVETimeFormat = "2006-01-02T15:04Z07:00"
+)
+
+type CVE struct {
+	CVEDataMeta struct {
+		ID string
+	} `json:"CVE_data_meta"`
+	References struct {
+		ReferenceData []struct {
+			URL string `json:"url"`
+		} `json:"reference_data"`
+	} `json:"references"`
+	Description struct {
+		DescriptionData []struct {
+			Lang  string `json:"lang"`
+			Value string `json:"value"`
+		} `json:"description_data"`
+	} `json:"description"`
+}
+
+type CVEItem struct {
+	CVE            CVE `json:"cve"`
+	Configurations struct {
+		Nodes []struct {
+			Operator string `json:"operator"`
+			CPEMatch []struct {
+				Vulnerable            bool   `json:"vulnerable"`
+				VersionStartExcluding string `json:"versionStartExcluding"`
+				VersionStartIncluding string `json:"versionStartIncluding"`
+				VersionEndExcluding   string `json:"versionEndExcluding"`
+				VersionEndIncluding   string `json:"versionEndIncluding"`
+			} `json:"cpe_match"`
+		} `json:"nodes"`
+	} `json:"configurations"`
+	Impact struct {
+		BaseMetricV3 struct {
+			CVSSV3 struct {
+				BaseSeverity string `json:"baseSeverity"`
+			} `json:"cvssV3"`
+		} `json:"baseMetricV3"`
+	} `json:"impact"`
+	PublishedDate    string `json:"publishedDate"`
+	LastModifiedDate string `json:"lastModifiedDate"`
+}
+
+type NVDCVE struct {
+	CVEItems         []CVEItem `json:"CVE_Items"`
+	CVEDataTimestamp string    `json:"CVE_data_timestamp"`
+}
+
+func EnglishDescription(cve CVE) string {
+	for _, desc := range cve.Description.DescriptionData {
+		if desc.Lang == "en" {
+			return desc.Value
+		}
+	}
+	return ""
+}
+
+func ParseTimestamp(timestamp string) (time.Time, error) {
+	return time.Parse(CVETimeFormat, timestamp)
+}

vulnfeeds/cves/versions.go

@@ -0,0 +1,100 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cves
+
+import (
+	"net/url"
+	"strings"
+)
+
+type FixCommit struct {
+	Repo   string
+	Commit string
+}
+
+type AffectedVersion struct {
+	Introduced string
+	Fixed      string
+}
+
+type VersionInfo struct {
+	FixCommits       []FixCommit
+	AffectedVersions []AffectedVersion
+}
+
+func extractGitHubCommit(link string) *FixCommit {
+	// Example: https://github.com/google/osv/commit/cd4e934d0527e5010e373e7fed54ef5daefba2f5
+	u, err := url.Parse(link)
+	if err != nil {
+		return nil
+	}
+
+	if u.Host != "github.com" {
+		return nil
+	}
+
+	pathParts := strings.Split(u.Path, "/")
+	if pathParts[len(pathParts)-2] != "commit" {
+		return nil
+	}
+
+	// Commit is the last component.
+	commit := pathParts[len(pathParts)-1]
+	// Stript the /commit/... to get the repo URL.
+	u.Path = strings.Join(pathParts[0:len(pathParts)-2], "/")
+	repo := u.String()
+
+	return &FixCommit{
+		Repo:   repo,
+		Commit: commit,
+	}
+}
+
+func ExtractVersionInfo(cve CVEItem) VersionInfo {
+	v := VersionInfo{}
+	for _, reference := range cve.CVE.References.ReferenceData {
+		// TODO(ochang): Support other common commit URLs.
+		if commit := extractGitHubCommit(reference.URL); commit != nil {
+			v.FixCommits = append(v.FixCommits, *commit)
+		}
+	}
+
+	for _, node := range cve.Configurations.Nodes {
+		if node.Operator != "OR" {
+			continue
+		}
+
+		// TODO: Also try to parse description as these are not always reliably set.
+		for _, match := range node.CPEMatch {
+			if !match.Vulnerable {
+				continue
+			}
+
+			if match.VersionStartIncluding != "" || match.VersionEndExcluding != "" {
+				if match.VersionStartExcluding != "" || match.VersionEndIncluding != "" {
+					// TODO: handle these.
+					continue
+				}
+
+				v.AffectedVersions = append(v.AffectedVersions, AffectedVersion{
+					// TODO: make sure these actually match PyPI numbers.
+					Introduced: match.VersionStartIncluding,
+					Fixed:      match.VersionEndExcluding,
+				})
+			}
+		}
+	}
+	return v
+}

vulnfeeds/go.mod

@@ -0,0 +1,5 @@
+module github.com/google/osv/vulnfeeds
+
+go 1.16
+
+require gopkg.in/yaml.v2 v2.4.0 // indirect

vulnfeeds/go.sum

@@ -0,0 +1,3 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=