Skip to content

Commit 571ee88

Browse files
timbastintimbastin
committed
adds content hash columns to malicious packages and exploits
1 parent cfadb36 commit 571ee88

9 files changed

Lines changed: 115 additions & 71 deletions

‎database/migrations/20260517000000_add_content_hash_to_exploits.up.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE public.exploits ADD COLUMN IF NOT EXISTS content_hash bigint NOT NULL DEFAULT 0;
2+
ALTER TABLE public.malicious_packages ADD COLUMN IF NOT EXISTS content_hash bigint NOT NULL DEFAULT 0;

‎database/models/cve_model.go‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99

1010
"github.com/l3montree-dev/devguard/dtos"
1111
"gorm.io/datatypes"
12-
"gorm.io/gorm"
1312
)
1413

1514
type Severity string
@@ -95,10 +94,3 @@ func (cve CVE) GetReferences() ([]cveReference, error) {
9594
return refs, nil
9695
}
9796

98-
func (cve *CVE) BeforeSave(tx *gorm.DB) error {
99-
if cve.ID == 0 {
100-
cve.ID = cve.CalculateHash()
101-
}
102-
cve.ContentHash = cve.CalculateContentHash()
103-
return nil
104-
}

‎database/models/exploit_model.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,16 @@
1414
// along with this program. If not, see <https://www.gnu.org/licenses/>.
1515
package models
1616

17-
import "time"
17+
import (
18+
"crypto/md5"
19+
"encoding/binary"
20+
"fmt"
21+
"time"
22+
)
1823

1924
type Exploit struct {
2025
ID string `json:"id" gorm:"primaryKey;"`
26+
ContentHash int64 `json:"contentHash" gorm:"type:bigint;not null;default:0;"`
2127
Published *time.Time `json:"pushed_at" gorm:"type:date;"`
2228
Updated *time.Time `json:"updated_at" gorm:"type:date;"`
2329
Author string `json:"author" gorm:"type:text;"`
@@ -34,6 +40,24 @@ type Exploit struct {
3440
Stars int `json:"stargazers_count" gorm:"type:integer;"`
3541
}
3642

43+
func (e Exploit) CalculateContentHash() int64 {
44+
pub := ""
45+
if e.Published != nil {
46+
pub = e.Published.Format(time.DateOnly)
47+
}
48+
upd := ""
49+
if e.Updated != nil {
50+
upd = e.Updated.Format(time.DateOnly)
51+
}
52+
h := fmt.Sprintf("%s|%s|%s|%s|%t|%s|%s|%s|%s|%d|%d|%d|%d",
53+
pub, upd, e.Author, e.Type, e.Verified, e.SourceURL, e.Description,
54+
e.CVEID, e.Tags, e.Forks, e.Watchers, e.Subscribers, e.Stars,
55+
)
56+
sum := md5.Sum([]byte(h))
57+
u := binary.BigEndian.Uint64(sum[:8])
58+
return int64(u & 0x7fffffffffffffff)
59+
}
60+
3761
func (m Exploit) TableName() string {
3862
return "exploits"
3963
}

‎database/models/malicious_package_model.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
package models
1717

1818
import (
19+
"crypto/md5"
1920
"crypto/sha256"
21+
"encoding/binary"
2022
"encoding/hex"
2123
"fmt"
2224
"time"
@@ -29,13 +31,25 @@ import (
2931
// MaliciousPackage stores metadata for malicious packages from OSV
3032
type MaliciousPackage struct {
3133
ID string `gorm:"primarykey;type:varchar(255)" json:"id"` // OSV ID
34+
ContentHash int64 `gorm:"type:bigint;not null;default:0" json:"contentHash"`
3235
Summary string `gorm:"type:text" json:"summary"`
3336
Details string `gorm:"type:text" json:"details"`
3437
Published time.Time `json:"published"`
3538
Modified time.Time `json:"modified"`
3639
MaliciousAffectedComponents []MaliciousAffectedComponent `json:"affectedComponents" gorm:"foreignKey:MaliciousPackageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
3740
}
3841

42+
func (mp MaliciousPackage) CalculateContentHash() int64 {
43+
h := fmt.Sprintf("%s|%s|%s|%s",
44+
mp.Summary, mp.Details,
45+
mp.Published.Format(time.RFC3339),
46+
mp.Modified.Format(time.RFC3339),
47+
)
48+
sum := md5.Sum([]byte(h))
49+
u := binary.BigEndian.Uint64(sum[:8])
50+
return int64(u & 0x7fffffffffffffff)
51+
}
52+
3953
func (MaliciousPackage) TableName() string {
4054
return "malicious_packages"
4155
}

‎vulndb/exploitdb_service.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,10 @@ func insertExploitsBulk(ctx context.Context, tx pgx.Tx, exploits []models.Exploi
107107
return nil
108108
}
109109
if _, err := tx.CopyFrom(ctx, pgx.Identifier{table},
110-
[]string{"id", "published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"},
110+
[]string{"id", "content_hash", "published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"},
111111
pgx.CopyFromSlice(len(exploits), func(i int) ([]any, error) {
112112
e := exploits[i]
113-
return []any{e.ID, e.Published, e.Updated, e.Author, e.Type, e.Verified, e.SourceURL, e.Description, e.CVEID, e.Tags, e.Forks, e.Watchers, e.Subscribers, e.Stars}, nil
113+
return []any{e.ID, e.CalculateContentHash(), e.Published, e.Updated, e.Author, e.Type, e.Verified, e.SourceURL, e.Description, e.CVEID, e.Tags, e.Forks, e.Watchers, e.Subscribers, e.Stars}, nil
114114
})); err != nil {
115115
return fmt.Errorf("could not copy exploit rows into staging table: %w", err)
116116
}

‎vulndb/gob.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ type CISAKEVEntry struct {
2929
// It omits the nested CVE field which contains datatypes.Date.
3030
type GobExploit struct {
3131
ID string
32+
ContentHash int64
3233
Published *time.Time
3334
Updated *time.Time
3435
Author string
@@ -91,8 +92,10 @@ func dateToTimePtr(d *datatypes.Date) *time.Time {
9192
// --- Exploit conversions ---
9293

9394
func exploitToGob(e models.Exploit) GobExploit {
95+
e.ContentHash = e.CalculateContentHash()
9496
return GobExploit{
9597
ID: e.ID,
98+
ContentHash: e.ContentHash,
9699
Published: e.Published,
97100
Updated: e.Updated,
98101
Author: e.Author,
@@ -112,6 +115,7 @@ func exploitToGob(e models.Exploit) GobExploit {
112115
func gobExploitToModel(g GobExploit) models.Exploit {
113116
return models.Exploit{
114117
ID: g.ID,
118+
ContentHash: g.ContentHash,
115119
Published: g.Published,
116120
Updated: g.Updated,
117121
Author: g.Author,

‎vulndb/malicious_packages_checker.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,10 @@ func (c *MaliciousPackageChecker) IsMalicious(ctx context.Context, ecosystem, pa
143143
func insertMaliciousPackagesBulk(ctx context.Context, tx pgx.Tx, pkgs []models.MaliciousPackage, comps []models.MaliciousAffectedComponent, pkgTable, compTable string) error {
144144
if len(pkgs) > 0 {
145145
if _, err := tx.CopyFrom(ctx, pgx.Identifier{pkgTable},
146-
[]string{"id", "summary", "details", "published", "modified"},
146+
[]string{"id", "content_hash", "summary", "details", "published", "modified"},
147147
pgx.CopyFromSlice(len(pkgs), func(i int) ([]any, error) {
148148
p := pkgs[i]
149-
return []any{p.ID, p.Summary, p.Details, p.Published, p.Modified}, nil
149+
return []any{p.ID, p.CalculateContentHash(), p.Summary, p.Details, p.Published, p.Modified}, nil
150150
})); err != nil {
151151
return fmt.Errorf("could not copy malicious packages into staging table: %w", err)
152152
}

‎vulndb/osv_service.go‎

Lines changed: 50 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ var liveTableSpecs = func() []syncSpec {
6969
acInsertCols := []string{"id", "purl", "ecosystem", "version", "semver_introduced", "semver_fixed", "version_introduced", "version_fixed"}
7070
acInsertExprs := []string{"id", "purl", "ecosystem", "version", "semver_introduced::semver", "semver_fixed::semver", "version_introduced", "version_fixed"}
7171
pivotAllCols := []string{"affected_component_id", "cve_id"}
72-
exploitAllCols := []string{"id", "published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"}
73-
malPkgAllCols := []string{"id", "summary", "details", "published", "modified"}
72+
exploitAllCols := []string{"id", "content_hash", "published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"}
73+
malPkgAllCols := []string{"id", "content_hash", "summary", "details", "published", "modified"}
7474
malCompInsertCols := []string{"id", "malicious_package_id", "purl", "ecosystem", "version", "semver_introduced", "semver_fixed", "version_introduced", "version_fixed"}
7575
malCompInsertExprs := []string{"id", "malicious_package_id", "purl", "ecosystem", "version::text", "semver_introduced::semver", "semver_fixed::semver", "version_introduced", "version_fixed"}
7676
return []syncSpec{
@@ -97,14 +97,14 @@ var liveTableSpecs = func() []syncSpec {
9797
},
9898
{
9999
live: "exploits", stage: "exploits_stage", keyCols: []string{"id"},
100-
contentHashCol: "updated",
101-
contentCols: []string{"published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"},
100+
contentHashCol: "content_hash",
101+
contentCols: []string{"content_hash", "published", "updated", "author", "type", "verified", "source_url", "description", "cve_id", "tags", "forks", "watchers", "subscribers", "stars"},
102102
insertCols: exploitAllCols, insertSelectExprs: exploitAllCols,
103103
},
104104
{
105105
live: "malicious_packages", stage: "mal_pkgs_stage", keyCols: []string{"id"},
106-
contentHashCol: "modified",
107-
contentCols: []string{"summary", "details", "published", "modified"},
106+
contentHashCol: "content_hash",
107+
contentCols: []string{"content_hash", "summary", "details", "published", "modified"},
108108
insertCols: malPkgAllCols, insertSelectExprs: malPkgAllCols,
109109
},
110110
{
@@ -730,13 +730,14 @@ func FlushOSVStagingTables(ctx context.Context, tx pgx.Tx) error {
730730

731731
t = time.Now()
732732
if _, err := tx.Exec(ctx, `
733-
INSERT INTO malicious_packages (id, summary, details, published, modified)
734-
SELECT id, summary, details, published, modified FROM mal_pkgs_stage
733+
INSERT INTO malicious_packages (id, content_hash, summary, details, published, modified)
734+
SELECT id, content_hash, summary, details, published, modified FROM mal_pkgs_stage
735735
ON CONFLICT (id) DO UPDATE SET
736-
summary = EXCLUDED.summary,
737-
details = EXCLUDED.details,
738-
published = EXCLUDED.published,
739-
modified = EXCLUDED.modified`); err != nil {
736+
content_hash = EXCLUDED.content_hash,
737+
summary = EXCLUDED.summary,
738+
details = EXCLUDED.details,
739+
published = EXCLUDED.published,
740+
modified = EXCLUDED.modified`); err != nil {
740741
return fmt.Errorf("could not flush malicious_packages: %w", err)
741742
}
742743
slog.Info("flushed malicious_packages", "took", time.Since(t))
@@ -768,19 +769,24 @@ func flushNonOSVStagingTables(ctx context.Context, tx pgx.Tx) error {
768769
return fmt.Errorf("could not delete stale exploits: %w", err)
769770
}
770771
if _, err := tx.Exec(ctx, `
771-
INSERT INTO exploits (id, published, updated, author, type, verified, source_url, description, cve_id, tags, forks, watchers, subscribers, stars)
772-
SELECT id, published, updated, author, type, verified, source_url, description, cve_id, tags, forks, watchers, subscribers, stars
772+
INSERT INTO exploits (id, content_hash, published, updated, author, type, verified, source_url, description, cve_id, tags, forks, watchers, subscribers, stars)
773+
SELECT id, content_hash, published, updated, author, type, verified, source_url, description, cve_id, tags, forks, watchers, subscribers, stars
773774
FROM exploits_stage
774775
ON CONFLICT (id) DO UPDATE SET
775-
published = EXCLUDED.published,
776-
updated = EXCLUDED.updated,
777-
author = EXCLUDED.author,
778-
source_url = EXCLUDED.source_url,
779-
description = EXCLUDED.description,
780-
forks = EXCLUDED.forks,
781-
watchers = EXCLUDED.watchers,
782-
subscribers = EXCLUDED.subscribers,
783-
stars = EXCLUDED.stars`); err != nil {
776+
content_hash = EXCLUDED.content_hash,
777+
published = EXCLUDED.published,
778+
updated = EXCLUDED.updated,
779+
author = EXCLUDED.author,
780+
type = EXCLUDED.type,
781+
verified = EXCLUDED.verified,
782+
source_url = EXCLUDED.source_url,
783+
description = EXCLUDED.description,
784+
cve_id = EXCLUDED.cve_id,
785+
tags = EXCLUDED.tags,
786+
forks = EXCLUDED.forks,
787+
watchers = EXCLUDED.watchers,
788+
subscribers = EXCLUDED.subscribers,
789+
stars = EXCLUDED.stars`); err != nil {
784790
return fmt.Errorf("could not flush exploits: %w", err)
785791
}
786792
slog.Info("finished flushing non-osv staging tables (exploits)", "total", time.Since(t))
@@ -830,28 +836,30 @@ func CreateStagingTables(ctx context.Context, tx pgx.Tx) error {
830836
) ON COMMIT DROP;
831837
832838
CREATE TEMP TABLE IF NOT EXISTS exploits_stage (
833-
id text,
834-
published date,
835-
updated date,
836-
author text,
837-
type text,
838-
verified boolean,
839-
source_url text,
840-
description text,
841-
cve_id text,
842-
tags text,
843-
forks integer,
844-
watchers integer,
845-
subscribers integer,
846-
stars integer
839+
id text,
840+
content_hash bigint,
841+
published date,
842+
updated date,
843+
author text,
844+
type text,
845+
verified boolean,
846+
source_url text,
847+
description text,
848+
cve_id text,
849+
tags text,
850+
forks integer,
851+
watchers integer,
852+
subscribers integer,
853+
stars integer
847854
) ON COMMIT DROP;
848855
849856
CREATE TEMP TABLE IF NOT EXISTS mal_pkgs_stage (
850-
id text,
851-
summary text,
852-
details text,
853-
published timestamptz,
854-
modified timestamptz
857+
id text,
858+
content_hash bigint,
859+
summary text,
860+
details text,
861+
published timestamptz,
862+
modified timestamptz
855863
) ON COMMIT DROP;
856864
857865
CREATE TEMP TABLE IF NOT EXISTS mal_comps_stage (

0 commit comments

Comments
 (0)