Skip to content

Commit c8bfec8

Browse files
themaroltthemarolt
committed
chore: import scorecards next to deps.dev data from openssf dataset
Signed-off-by: Uroš Marolt <uros@marolt.me>
1 parent 9867df3 commit c8bfec8

15 files changed

Lines changed: 294 additions & 16 deletions

File tree

‎backend/src/osspckgs/migrations/V1781074345__add-scorecard-job-kinds.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-- Extend osspckgs_ingest_jobs.job_kind CHECK constraint to include scorecard kinds.
2+
-- Required for ingestScorecard workflow (CM-1227).
3+
ALTER TABLE osspckgs_ingest_jobs
4+
DROP CONSTRAINT osspckgs_ingest_jobs_job_kind_check,
5+
ADD CONSTRAINT osspckgs_ingest_jobs_job_kind_check CHECK (job_kind IN (
6+
'packages', 'versions', 'package_dependencies',
7+
'repos', 'package_repos',
8+
'advisories', 'advisory_packages',
9+
'dependent_counts',
10+
'scorecard_repos', 'scorecard_checks'
11+
));

scripts/services/deps-dev-ingest.yaml renamed to scripts/services/bq-dataset-ingest.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,18 @@ version: '3.1'
33
x-env-args: &env-args
44
DOCKER_BUILDKIT: 1
55
NODE_ENV: docker
6-
SERVICE: deps-dev-ingest
7-
CROWD_TEMPORAL_TASKQUEUE: deps-dev-ingest
6+
SERVICE: bq-dataset-ingest
7+
CROWD_TEMPORAL_TASKQUEUE: bq-dataset-ingest
88
CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE}
99
SHELL: /bin/sh
1010
SUPPRESS_NO_CONFIG_WARNING: 'true'
1111

1212
services:
13-
deps-dev-ingest:
13+
bq-dataset-ingest:
1414
build:
1515
context: ../../
1616
dockerfile: ./scripts/services/docker/Dockerfile.packages
17-
command: 'pnpm run start:deps-dev-ingest'
17+
command: 'pnpm run start:bq-dataset-ingest'
1818
working_dir: /usr/crowd/app/services/apps/packages_worker
1919
env_file:
2020
- ../../backend/.env.dist.local
@@ -27,11 +27,11 @@ services:
2727
networks:
2828
- crowd-bridge
2929

30-
deps-dev-ingest-dev:
30+
bq-dataset-ingest-dev:
3131
build:
3232
context: ../../
3333
dockerfile: ./scripts/services/docker/Dockerfile.packages
34-
command: 'pnpm run dev:deps-dev-ingest'
34+
command: 'pnpm run dev:bq-dataset-ingest'
3535
working_dir: /usr/crowd/app/services/apps/packages_worker
3636
# user: '${USER_ID}:${GROUP_ID}'
3737
env_file:
@@ -41,7 +41,7 @@ services:
4141
- ../../backend/.env.override.composed
4242
environment:
4343
<<: *env-args
44-
hostname: deps-dev-ingest
44+
hostname: bq-dataset-ingest
4545
networks:
4646
- crowd-bridge
4747
volumes:

‎services/apps/packages_worker/package.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"scripts": {
55
"start:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker tsx src/bin/packages-worker.ts",
66
"start:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker tsx src/bin/criticality-worker.ts",
7-
"start:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest tsx src/bin/deps-dev-ingest.ts",
7+
"start:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest tsx src/bin/bq-dataset-ingest.ts",
88
"start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker tsx src/bin/npm-worker.ts",
99
"start:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker tsx src/bin/osv-worker.ts",
1010
"start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
@@ -18,21 +18,21 @@
1818
"start:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker tsx src/bin/maven-worker.ts",
1919
"backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
2020
"dev:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
21-
"dev:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
21+
"dev:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
2222
"dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
2323
"dev:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
2424
"dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
2525
"dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
2626
"dev:criticality-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
2727
"dev:maven-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
28-
"dev:deps-dev-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
28+
"dev:bq-dataset-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
2929
"dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
3030
"dev:osv-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
3131
"start:maven": "SERVICE=maven tsx src/bin/maven.ts",
3232
"dev:maven": "SERVICE=maven LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/maven.ts",
3333
"dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
34-
"export-to-bucket": "SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
35-
"export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
34+
"export-to-bucket": "SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
35+
"export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
3636
"monitor:osspckgs:local": "bash -c 'set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && node ../../../scripts/monitor-osspckgs.mjs'",
3737
"lint": "npx eslint --ext .ts src --max-warnings=0",
3838
"format": "npx prettier --write \"src/**/*.ts\"",

services/apps/packages_worker/src/bin/deps-dev-ingest.ts renamed to services/apps/packages_worker/src/bin/bq-dataset-ingest.ts

File renamed without changes.

‎services/apps/packages_worker/src/deps-dev/config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ function requireEnv(name: string): string {
1010
export const GCP_PROJECT = requireEnv('OSSPCKGS_GCP_PROJECT')
1111
export const GCS_BUCKET = requireEnv('OSSPCKGS_GCS_BUCKET')
1212
export const DEPS_DEV_DATASET = 'bigquery-public-data.deps_dev_v1'
13+
export const SCORECARD_DATASET = 'openssf.scorecardcron'
1314

1415
// ADR-0003: Option A = DependencyGraphEdgesLatest (prod default, has version_constraint).
1516
// Set OSSPCKGS_DEPS_TABLE=B locally to use DependenciesLatest (cheaper, no version_constraint).

‎services/apps/packages_worker/src/deps-dev/schedules/bootstrap.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export async function scheduleOsspckgsBootstrap(): Promise<void> {
1111
await temporal.schedule.create({
1212
scheduleId: 'osspckgs-bootstrap-weekly',
1313
spec: {
14-
cronExpressions: ['0 2 * * 0'],
14+
cronExpressions: ['0 2 * * 1'],
1515
},
1616
policies: {
1717
overlap: ScheduleOverlapPolicy.SKIP,
@@ -20,7 +20,7 @@ export async function scheduleOsspckgsBootstrap(): Promise<void> {
2020
action: {
2121
type: 'startWorkflow',
2222
workflowType: bootstrapOsspckgs,
23-
taskQueue: 'deps-dev-ingest',
23+
taskQueue: 'bq-dataset-ingest',
2424
workflowExecutionTimeout: '12 hours',
2525
retry: {
2626
initialInterval: '1 minute',

‎services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { ingestDependentCounts } from './ingestDependentCounts'
1313
import { ingestPackages } from './ingestPackages'
1414
import { ingestRepos } from './ingestRepos'
1515
import { ingestVersions } from './ingestVersions'
16+
import { ingestScorecard } from '../../scorecard/workflows'
1617

1718
const { getLastSnapshot, probePartitionExists, resolveSnapshotDate } = proxyActivities<
1819
typeof depsDevActivities
@@ -224,4 +225,9 @@ export async function bootstrapOsspckgs(opts: {
224225
],
225226
})
226227
}
228+
if (runs('scorecard')) {
229+
await executeChild(ingestScorecard, {
230+
args: [{ runId, reuseExports: opts.reuseExports, exportName: opts.exportName }],
231+
})
232+
}
227233
}

‎services/apps/packages_worker/src/schedules/cleanup.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export async function scheduleOsspckgsCleanup(): Promise<void> {
2020
action: {
2121
type: 'startWorkflow',
2222
workflowType: cleanupOsspckgs,
23-
taskQueue: 'deps-dev-ingest',
23+
taskQueue: 'bq-dataset-ingest',
2424
workflowExecutionTimeout: '1 hour',
2525
retry: {
2626
initialInterval: '1 minute',

‎services/apps/packages_worker/src/scorecard/queries/scorecardSql.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { SCORECARD_DATASET } from '../../deps-dev/config'
2+
3+
export const SCORECARD_REPOS_SQL = `
4+
SELECT
5+
CONCAT('https://', repo.name) AS repo_url,
6+
score,
7+
date AS scanned_at
8+
FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\`
9+
WHERE repo.name IS NOT NULL
10+
`
11+
12+
export const SCORECARD_CHECKS_SQL = `
13+
SELECT
14+
CONCAT('https://', r.repo.name) AS repo_url,
15+
c.name AS check_name,
16+
c.score AS check_score,
17+
c.reason AS check_reason
18+
FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\` r,
19+
UNNEST(r.checks) AS c
20+
WHERE r.repo.name IS NOT NULL
21+
`

‎services/apps/packages_worker/src/scorecard/workflows/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export { ingestScorecard } from './ingestScorecard'

0 commit comments

Comments
 (0)